[Mini-vLLM] SM86-only cut: models, backends, Rust, tests

- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries
- Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels
- Remove inc/quark quant loaders; keep compressed-tensors and Marlin
- Drop the Rust frontend, pooling entrypoints, and idle test suites
- Rebuild kernels for sm_80;sm_86 and strip the extensions
- Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-18 19:19:56 +07:00
co-authored by Hinata
parent c5ab42bc0f
commit e61dcb9143
2728 changed files with 159 additions and 950401 deletions
-471
View File
@@ -1,471 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness tests for the AMX-only CPU MLA backend: the vendored
decode/extend/bmm kernels, the KV cache write, and the ``AMXMLAImpl``
backend built on top of them.
"""
import pytest
import torch
from vllm.platforms import current_platform
if not current_platform.is_cpu():
pytest.skip("skipping CPU-only tests", allow_module_level=True)
if not torch.cpu._is_amx_tile_supported():
pytest.skip("AMX MLA requires an AMX-capable host", allow_module_level=True)
torch.cpu._init_amx()
from vllm import _custom_ops as ops # noqa: E402
from vllm.utils.torch_utils import set_random_seed # noqa: E402
from vllm.v1.attention.backends.mla.amx_mla import ( # noqa: E402
AMXMLAImpl,
_compute_num_kv_splits,
_expand_block_table,
)
KV_LORA_RANK = 64
QK_ROPE_HEAD_DIM = 32
HEAD_SIZE = KV_LORA_RANK + QK_ROPE_HEAD_DIM # 96, kv-cache row width
# The extend/decode kernels transpose by 512-bit lanes: both the cache row
# width and the (kv_lora_rank-wide) value width must be 32-element aligned.
DTYPE = torch.bfloat16
ATOL = 2e-2
RTOL = 2e-2
# Backend-level tests chain absorb -> attend -> de-absorb through bf16,
# accumulating more rounding error than a single kernel op.
_IMPL_ATOL = 1.5e-1
_IMPL_RTOL = 1.5e-1
def _flatten_cache(kv_cache: torch.Tensor) -> torch.Tensor:
"""(num_blocks, block_size, head_size) -> (num_slots, head_size)."""
return kv_cache.view(-1, kv_cache.size(-1))
def _random_paged_cache(num_slots: int, head_size: int) -> torch.Tensor:
return torch.randn(num_slots, head_size, dtype=DTYPE)
def _ref_latent_attn(
q: torch.Tensor,
keys: torch.Tensor,
scale: float,
kv_lora_rank: int,
) -> torch.Tensor:
"""Plain-PyTorch causal-free latent-space MQA attention for one request.
q: (num_heads, head_size); keys: (ctx_len, head_size). Values are the
first ``kv_lora_rank`` columns of the same latent rows (MLA's K/V
aliasing), matching what decode_attention_cpu/extend_attention_cpu
compute internally for the MLA-shaped case.
"""
values = keys[:, :kv_lora_rank]
scores = (q.float() @ keys.float().T) * scale
probs = torch.softmax(scores, dim=-1)
return (probs @ values.float()).to(q.dtype)
def test_bmm_cpu_matches_torch_bmm():
set_random_seed(0)
# bmm_cpu requires the output's last dim (mat2's out-features) to be a
# multiple of 32 (tinygemm's tile width); the contraction dim is free.
n, b, p, l = 8, 32, 32, 32 # noqa: E741
mat1 = torch.randn(n, b, p, dtype=DTYPE)
mat2 = torch.randn(n, l, p, dtype=DTYPE) # (N, OUT, IN) Linear convention
ref = torch.bmm(mat1.float(), mat2.float().transpose(1, 2)).to(DTYPE)
out = torch.empty(n, b, l, dtype=DTYPE)
ops.bmm_cpu(out, mat1, mat2, False, None)
torch.testing.assert_close(out, ref, atol=ATOL, rtol=RTOL)
packed = torch.ops._C.convert_weight_packed(mat2.contiguous())
out_vnni = torch.empty(n, b, l, dtype=DTYPE)
ops.bmm_cpu(out_vnni, mat1, packed, True, None)
torch.testing.assert_close(out_vnni, ref, atol=ATOL, rtol=RTOL)
def test_concat_and_cache_mla_round_trip():
set_random_seed(0)
num_tokens = 37
num_blocks, block_size = 8, 16
kv_cache = torch.zeros(num_blocks, block_size, HEAD_SIZE, dtype=DTYPE)
kv_c_normed = torch.randn(num_tokens, KV_LORA_RANK, dtype=DTYPE)
k_pe = torch.randn(num_tokens, QK_ROPE_HEAD_DIM, dtype=DTYPE)
slot_mapping = torch.randperm(num_blocks * block_size)[:num_tokens].to(torch.int64)
ops.amx_mla_concat_and_cache(kv_c_normed, k_pe, kv_cache, slot_mapping)
flat = _flatten_cache(kv_cache)
read_back = flat[slot_mapping]
expected = torch.cat([kv_c_normed, k_pe], dim=-1)
torch.testing.assert_close(read_back, expected)
untouched = torch.ones(num_blocks * block_size, dtype=torch.bool)
untouched[slot_mapping] = False
assert (flat[untouched] == 0).all()
@pytest.mark.parametrize("seq_lens", [[5], [1, 300, 33], [513, 1, 129, 7]])
def test_decode_attention_cpu_matches_reference(seq_lens):
set_random_seed(0)
num_seqs = len(seq_lens)
num_heads = 4
block_size = 32
max_len = max(seq_lens)
max_blocks = (max_len + block_size - 1) // block_size
num_blocks = num_seqs * max_blocks
kv_cache = _random_paged_cache(num_blocks * block_size, HEAD_SIZE)
block_table = torch.randperm(num_blocks).view(num_seqs, max_blocks).to(torch.int64)
req_to_token = _expand_block_table(block_table, block_size)
req_pool_indices = torch.arange(num_seqs, dtype=torch.int64)
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int64)
scale = HEAD_SIZE**-0.5
q = torch.randn(num_seqs, num_heads, HEAD_SIZE, dtype=DTYPE)
kv_cache_flat = kv_cache.view(-1, 1, HEAD_SIZE)
v_buffer = kv_cache_flat[..., :KV_LORA_RANK]
o = torch.zeros(num_seqs, num_heads, KV_LORA_RANK, dtype=DTYPE)
attn_logits = torch.zeros(
num_seqs, num_heads, 4, KV_LORA_RANK + 1, dtype=torch.float32
)
ops.cpu_mla_decode(
q,
kv_cache_flat,
v_buffer,
o,
None,
None,
None,
attn_logits,
req_to_token,
req_pool_indices,
seq_lens_t,
scale,
0.0,
False,
0,
None,
None,
)
flat_cache = _flatten_cache(kv_cache)
for i, seq_len in enumerate(seq_lens):
token_ids = req_to_token[i, :seq_len]
keys = flat_cache[token_ids]
ref = _ref_latent_attn(q[i], keys, scale, KV_LORA_RANK)
torch.testing.assert_close(o[i], ref, atol=ATOL, rtol=RTOL)
@pytest.mark.parametrize(
"prefix_lens,extend_lens",
[
([0], [9]), # fresh prefill, no cached prefix
([64], [17]), # continuation of a cached prefix
([0, 40, 128], [23, 5, 31]), # mixed batch
],
)
def test_extend_attention_cpu_matches_reference(prefix_lens, extend_lens):
set_random_seed(0)
num_seqs = len(prefix_lens)
num_heads = 4
block_size = 32
seq_lens = [p + e for p, e in zip(prefix_lens, extend_lens)]
max_len = max(seq_lens)
max_blocks = (max_len + block_size - 1) // block_size
num_blocks = num_seqs * max_blocks
kv_cache = _random_paged_cache(num_blocks * block_size, HEAD_SIZE)
block_table = torch.randperm(num_blocks).view(num_seqs, max_blocks).to(torch.int64)
req_to_token = _expand_block_table(block_table, block_size)
req_pool_indices = torch.arange(num_seqs, dtype=torch.int64)
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int64)
extend_seq_lens_t = torch.tensor(extend_lens, dtype=torch.int64)
extend_start_loc = torch.cumsum(
torch.tensor([0, *extend_lens[:-1]], dtype=torch.int64), dim=0
)
scale = HEAD_SIZE**-0.5
total_new_tokens = sum(extend_lens)
# The new tokens' latent K/V, already written into the cache (mirrors
# do_kv_cache_update running before forward_mha in real usage).
k_extend_flat = torch.randn(total_new_tokens, HEAD_SIZE, dtype=DTYPE)
flat_cache = _flatten_cache(kv_cache)
for i in range(num_seqs):
prefix_len = prefix_lens[i]
ext_len = extend_lens[i]
new_token_ids = req_to_token[i, prefix_len : prefix_len + ext_len]
start = extend_start_loc[i].item()
flat_cache[new_token_ids] = k_extend_flat[start : start + ext_len]
q_extend = torch.randn(total_new_tokens, num_heads, HEAD_SIZE, dtype=DTYPE)
k_extend = k_extend_flat.unsqueeze(1)
v_extend = k_extend[..., :KV_LORA_RANK]
kv_cache_flat = kv_cache.view(-1, 1, HEAD_SIZE)
v_buffer = kv_cache_flat[..., :KV_LORA_RANK]
o_extend = torch.empty(total_new_tokens, num_heads, KV_LORA_RANK, dtype=DTYPE)
ops.cpu_mla_extend(
q_extend,
k_extend,
v_extend,
o_extend,
kv_cache_flat,
v_buffer,
req_to_token,
req_pool_indices,
seq_lens_t,
extend_seq_lens_t,
extend_start_loc,
max(extend_lens),
scale,
0.0,
False,
0,
None,
None,
None,
)
for i in range(num_seqs):
prefix_len = prefix_lens[i]
ext_len = extend_lens[i]
start = extend_start_loc[i].item()
for t in range(ext_len):
visible = prefix_len + t + 1
token_ids = req_to_token[i, :visible]
keys = flat_cache[token_ids]
ref = _ref_latent_attn(q_extend[start + t], keys, scale, KV_LORA_RANK)
torch.testing.assert_close(o_extend[start + t], ref, atol=ATOL, rtol=RTOL)
class _FakeLinear:
"""Minimal duck-typed stand-in for kv_b_proj: only .weight and
.quant_method are read by get_and_maybe_dequant_weights()."""
def __init__(self, weight: torch.Tensor):
self.weight = weight
self.quant_method = None
def _make_amx_mla_impl(num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank):
weight = torch.randn(
num_heads * (qk_nope_head_dim + v_head_dim), kv_lora_rank, dtype=DTYPE
)
kv_b_proj = _FakeLinear(weight)
impl = AMXMLAImpl(
num_heads=num_heads,
head_size=kv_lora_rank + QK_ROPE_HEAD_DIM,
scale=(qk_nope_head_dim + QK_ROPE_HEAD_DIM) ** -0.5,
num_kv_heads=1,
alibi_slopes=None,
sliding_window=None,
kv_cache_dtype="auto",
logits_soft_cap=None,
attn_type="decoder",
kv_sharing_target_layer_name=None,
q_lora_rank=None,
kv_lora_rank=kv_lora_rank,
qk_nope_head_dim=qk_nope_head_dim,
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
qk_head_dim=qk_nope_head_dim + QK_ROPE_HEAD_DIM,
v_head_dim=v_head_dim,
kv_b_proj=kv_b_proj,
)
impl.process_weights_after_loading(DTYPE)
return impl, weight
class _FakePrefillMetadata:
def __init__(
self, block_table, cpu_seq_lens, query_start_loc, req_to_token, req_pool_indices
):
self.block_table = block_table
self.cpu_seq_lens = cpu_seq_lens
query_start_loc_i64 = query_start_loc.to(torch.int64)
extend_seq_lens = query_start_loc_i64[1:] - query_start_loc_i64[:-1]
self.extend_seq_lens = extend_seq_lens
self.extend_start_loc = query_start_loc_i64[:-1]
self.max_len_extend = int(extend_seq_lens.max().item())
self.req_to_token = req_to_token
self.req_pool_indices = req_pool_indices
class _FakeDecodeMetadata:
def __init__(self, block_table, seq_lens, req_to_token, req_pool_indices):
self.block_table = block_table
self.seq_lens_i64 = seq_lens.to(torch.int64)
self.req_to_token = req_to_token
self.req_pool_indices = req_pool_indices
self.num_kv_splits = _compute_num_kv_splits(
int(seq_lens.max().item()), current_platform.num_compute_units()
)
class _FakeAttnMetadata:
def __init__(self, decode=None, prefill=None, max_seq_len=0):
self.decode = decode
self.prefill = prefill
self.max_seq_len = max_seq_len
def test_amx_mla_impl_forward_mqa_matches_reference(default_vllm_config):
"""forward_mqa receives already-absorbed Q (as MLAAttention.forward_impl
would produce via layer.W_UK_T) and must reproduce plain per-head MLA
decode attention through W_UK/W_UV."""
set_random_seed(1)
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank = 4, 32, 32, 64
impl, kv_b_weight = _make_amx_mla_impl(
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank
)
w_uk, w_uv = kv_b_weight.T.view(
kv_lora_rank, num_heads, qk_nope_head_dim + v_head_dim
).split([qk_nope_head_dim, v_head_dim], dim=-1)
seq_lens = [1, 40, 5]
num_seqs = len(seq_lens)
block_size = 32
max_blocks = (max(seq_lens) + block_size - 1) // block_size
num_blocks = num_seqs * max_blocks
head_size = kv_lora_rank + QK_ROPE_HEAD_DIM
kv_cache = torch.randn(num_blocks, block_size, head_size, dtype=DTYPE)
block_table = torch.randperm(num_blocks).view(num_seqs, max_blocks).to(torch.int64)
req_to_token = _expand_block_table(block_table, block_size)
q_nope = torch.randn(num_seqs, num_heads, qk_nope_head_dim, dtype=DTYPE)
q_pe = torch.randn(num_seqs, num_heads, QK_ROPE_HEAD_DIM, dtype=DTYPE)
# Pre-absorb, mirroring MLAAttention.forward_impl's own bmm against W_UK_T.
ql_nope = torch.einsum("bnp,lnp->bnl", q_nope.float(), w_uk.float()).to(DTYPE)
attn_metadata = _FakeAttnMetadata(
decode=_FakeDecodeMetadata(
block_table=block_table,
seq_lens=torch.tensor(seq_lens, dtype=torch.int64),
req_to_token=req_to_token,
req_pool_indices=torch.arange(num_seqs, dtype=torch.int64),
),
max_seq_len=max(seq_lens),
)
o, lse = impl.forward_mqa((ql_nope, q_pe), kv_cache, attn_metadata, layer=None)
assert lse is None
# De-absorb, mirroring MLAAttention.forward_impl's own bmm against
# layer._v_up_proj, to get back into real per-head V space for
# comparison against a from-scratch reference.
out_real = torch.einsum("bnl,lnv->bnv", o.float(), w_uv.float())
flat_cache = _flatten_cache(kv_cache)
scale = impl.scale
for i, seq_len in enumerate(seq_lens):
token_ids = req_to_token[i, :seq_len]
keys_latent = flat_cache[token_ids] # (L, head_size)
k_nope_latent, k_pe = keys_latent.split(
[kv_lora_rank, QK_ROPE_HEAD_DIM], dim=-1
)
k_real = torch.einsum("lp,pnd->nld", k_nope_latent.float(), w_uk.float())
k_pe_b = k_pe.float().unsqueeze(0).expand(num_heads, -1, -1)
k_full = torch.cat([k_real, k_pe_b], dim=-1) # (num_heads, L, qk_head_dim)
q_full = torch.cat(
[q_nope[i], q_pe[i]], dim=-1
).float() # (num_heads, qk_head_dim)
scores = torch.einsum("nd,nld->nl", q_full, k_full) * scale
probs = torch.softmax(scores, dim=-1)
v_real = torch.einsum("lp,pnv->nlv", k_nope_latent.float(), w_uv.float())
ref = torch.einsum("nl,nlv->nv", probs, v_real)
torch.testing.assert_close(out_real[i], ref, atol=_IMPL_ATOL, rtol=_IMPL_RTOL)
def test_amx_mla_impl_forward_mha_matches_reference(default_vllm_config):
"""forward_mha receives raw unabsorbed Q and must attend correctly for a
prefill batch with a mix of fresh and continued (cached-prefix)
sequences."""
set_random_seed(2)
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank = 4, 32, 32, 64
impl, kv_b_weight = _make_amx_mla_impl(
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank
)
w_uk, w_uv = kv_b_weight.T.view(
kv_lora_rank, num_heads, qk_nope_head_dim + v_head_dim
).split([qk_nope_head_dim, v_head_dim], dim=-1)
prefix_lens = [0, 20]
extend_lens = [13, 7]
seq_lens = [p + e for p, e in zip(prefix_lens, extend_lens)]
num_seqs = len(prefix_lens)
block_size = 32
max_blocks = (max(seq_lens) + block_size - 1) // block_size
num_blocks = num_seqs * max_blocks
head_size = kv_lora_rank + QK_ROPE_HEAD_DIM
kv_cache = torch.randn(num_blocks, block_size, head_size, dtype=DTYPE)
block_table = torch.randperm(num_blocks).view(num_seqs, max_blocks).to(torch.int64)
req_to_token = _expand_block_table(block_table, block_size)
flat_cache = _flatten_cache(kv_cache)
total_new_tokens = sum(extend_lens)
query_start_loc = torch.tensor(
[0, *torch.cumsum(torch.tensor(extend_lens), dim=0).tolist()],
dtype=torch.int64,
)
q_nope = torch.randn(total_new_tokens, num_heads, qk_nope_head_dim, dtype=DTYPE)
q_pe = torch.randn(total_new_tokens, num_heads, QK_ROPE_HEAD_DIM, dtype=DTYPE)
q = torch.cat([q_nope, q_pe], dim=-1)
kv_c_normed = torch.randn(total_new_tokens, kv_lora_rank, dtype=DTYPE)
k_pe = torch.randn(total_new_tokens, 1, QK_ROPE_HEAD_DIM, dtype=DTYPE)
# Write the new tokens' latent K/V into the cache ahead of time, mirroring
# do_kv_cache_update running before forward_mha in real usage.
for i in range(num_seqs):
prefix_len = prefix_lens[i]
ext_len = extend_lens[i]
new_token_ids = req_to_token[i, prefix_len : prefix_len + ext_len]
start = query_start_loc[i].item()
flat_cache[new_token_ids, :kv_lora_rank] = kv_c_normed[start : start + ext_len]
flat_cache[new_token_ids, kv_lora_rank:] = k_pe[start : start + ext_len, 0]
attn_metadata = _FakeAttnMetadata(
prefill=_FakePrefillMetadata(
block_table=block_table,
cpu_seq_lens=torch.tensor(seq_lens, dtype=torch.int64),
query_start_loc=query_start_loc,
req_to_token=req_to_token,
req_pool_indices=torch.arange(num_seqs, dtype=torch.int64),
)
)
output = torch.empty(total_new_tokens, num_heads * v_head_dim, dtype=DTYPE)
impl.forward_mha(
q, kv_c_normed, k_pe, kv_cache, attn_metadata, k_scale=None, output=output
)
out_view = output.view(total_new_tokens, num_heads, v_head_dim).float()
scale = impl.scale
for i in range(num_seqs):
prefix_len = prefix_lens[i]
ext_len = extend_lens[i]
start = query_start_loc[i].item()
for t in range(ext_len):
visible = prefix_len + t + 1
token_ids = req_to_token[i, :visible]
keys_latent = flat_cache[token_ids]
k_nope_latent, k_pe_ref = keys_latent.split(
[kv_lora_rank, QK_ROPE_HEAD_DIM], dim=-1
)
k_real = torch.einsum("lp,pnd->nld", k_nope_latent.float(), w_uk.float())
k_pe_b = k_pe_ref.float().unsqueeze(0).expand(num_heads, -1, -1)
k_full = torch.cat([k_real, k_pe_b], dim=-1)
q_full = torch.cat([q_nope[start + t], q_pe[start + t]], dim=-1).float()
scores = torch.einsum("nd,nld->nl", q_full, k_full) * scale
probs = torch.softmax(scores, dim=-1)
v_real = torch.einsum("lp,pnv->nlv", k_nope_latent.float(), w_uv.float())
ref = torch.einsum("nl,nlv->nv", probs, v_real)
torch.testing.assert_close(
out_view[start + t], ref, atol=_IMPL_ATOL, rtol=_IMPL_RTOL
)
@@ -1,919 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from contextlib import contextmanager
from typing import Literal
from unittest.mock import MagicMock, patch
import pytest
import torch
from vllm.config import (
AttentionConfig,
CacheConfig,
ParallelConfig,
VllmConfig,
set_current_vllm_config,
)
from vllm.config.cache import CacheDType
from vllm.platforms import current_platform
from vllm.platforms.cpu import CpuPlatform
from vllm.platforms.interface import DeviceCapability
if current_platform.is_cuda():
from vllm.platforms.cuda import CudaPlatform
else:
CudaPlatform = None
if current_platform.is_rocm():
from vllm.platforms.rocm import RocmPlatform
else:
RocmPlatform = None # type: ignore[misc] # Unavailable platform import.
from vllm.v1.attention.backend import AttentionType
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.attention.selector import (
AttentionSelectorConfig,
_cached_get_attn_backend,
get_attn_backend,
)
@pytest.fixture(autouse=True)
def clear_cache():
"""Clear lru cache to ensure each test case runs without caching."""
_cached_get_attn_backend.cache_clear()
# Define MLA and non-MLA backends separately
DEVICE_MLA_BACKENDS = {
"cuda": [
"TRITON_MLA",
"FLASHMLA",
"FLASHINFER_MLA",
"FLASH_ATTN_MLA",
"CUTLASS_MLA",
],
"hip": ["TRITON_MLA", "ROCM_AITER_MLA"],
"cpu": [],
}
DEVICE_REGULAR_ATTN_BACKENDS = {
"cuda": ["FLASHINFER", "FLASH_ATTN"],
"hip": ["ROCM_ATTN"],
"cpu": ["CPU_ATTN"],
}
DEVICE_MLA_BLOCK_SIZES = {
"cuda": [16, 64], # CUDA supports both standard and extended block sizes
"hip": [16, 1], # HIP requires special handling for block_size=1
# "cpu": [16] # CPU uses fixed block size from test cases
"cpu": [], # FIXME(woosuk): Temporarily disable CPU tests
}
def generate_params():
is_rocm = current_platform.is_rocm()
params = []
device_list = ["cuda", "cpu"] if not is_rocm else ["hip", "cpu"]
for use_mla in [True, False]:
for device in device_list:
backends = (
DEVICE_MLA_BACKENDS[device]
if use_mla
else DEVICE_REGULAR_ATTN_BACKENDS[device]
)
for name in backends:
block_sizes = (
[128]
if name == "CUTLASS_MLA"
else DEVICE_MLA_BLOCK_SIZES[device]
if use_mla
else [16]
)
for block_size in block_sizes:
for use_dcp in [False, True] if device != "cpu" else [False]:
params.append(
pytest.param(
device,
name,
use_mla,
block_size,
use_dcp,
id=(
f"{device}_{name}_mla_{str(use_mla)[0]}"
f"_blks{block_size}_dcp{use_dcp}"
),
)
)
return params
@pytest.mark.parametrize(
"device, name, use_mla, block_size, use_dcp", generate_params()
)
def test_backend_selection(
device: str,
name: str,
use_mla: bool,
block_size: int,
use_dcp: bool,
):
"""Supported GPU backends remain selectable when DCP is enabled."""
# Create AttentionConfig with the specified backend
attention_config = AttentionConfig(backend=AttentionBackendEnum[name])
cache_config = CacheConfig(block_size=block_size)
vllm_config = VllmConfig(
attention_config=attention_config,
cache_config=cache_config,
parallel_config=ParallelConfig(
# Skip GPU-count autodetection: selection does not launch workers.
distributed_executor_backend="mp" if use_dcp else None,
tensor_parallel_size=2 if use_dcp else 1,
decode_context_parallel_size=2 if use_dcp else 1,
),
)
with set_current_vllm_config(vllm_config):
if device == "cpu":
with patch("vllm.platforms.current_platform", CpuPlatform()):
backend = get_attn_backend(16, torch.float16, None)
assert backend.get_name() == "CPU_ATTN"
elif device == "hip":
if RocmPlatform is None:
pytest.skip("RocmPlatform not available")
with patch("vllm.platforms.current_platform", RocmPlatform()):
if use_mla:
# ROCm MLA backend logic:
# - TRITON_MLA: supported when block_size != 1
# - ROCM_AITER_MLA: supported when block_size == 1
# If backend is forced but doesn't match block_size,
# should raise ValueError
if name == "TRITON_MLA" and block_size == 1:
# TRITON_MLA doesn't support block_size == 1
with pytest.raises(ValueError):
get_attn_backend(576, torch.float16, None, use_mla=use_mla)
else:
# Valid backend-block_size combination
backend = get_attn_backend(
576, torch.float16, None, use_mla=use_mla
)
expected = name
assert backend.get_name() == expected
else:
if use_dcp:
with pytest.raises(ValueError, match="DCP not supported"):
get_attn_backend(32, torch.float16, None, use_mla=use_mla)
else:
backend = get_attn_backend(
32, torch.float16, None, use_mla=use_mla
)
expected = "ROCM_ATTN"
assert backend.get_name() == expected
elif device == "cuda":
if CudaPlatform is None:
pytest.skip("CudaPlatform not available")
with patch("vllm.platforms.current_platform", CudaPlatform()):
capability = torch.cuda.get_device_capability()
if use_mla:
# CUDA MLA backend logic:
# - CUTLASS_MLA: only supported with block_size == 128
# and Blackwell GPUs (SM 10.x), V1 only
# - FLASHINFER_MLA: only supported on Blackwell GPUs
# (SM 10.x), V1 only
# - FLASHMLA: only supported with block_size == 64
# - FLASH_ATTN_MLA: V1 only
# - TRITON_MLA: fallback for other cases
if name == "CUTLASS_MLA":
if block_size != 128:
# CUTLASS_MLA only supports block_size == 128
pytest.skip("CUTLASS_MLA only supports block_size 128")
if capability[0] != 10:
pytest.skip("CUTLASS MLA is not supported on this platform")
backend = get_attn_backend(
576, torch.float16, None, use_mla=use_mla
)
expected = "CUTLASS_MLA"
assert backend.get_name() == expected
elif name == "FLASHINFER_MLA":
if capability[0] != 10:
pytest.skip(
"FlashInfer MLA is not supported on this platform"
)
if block_size not in [32, 64]:
# FlashInfer MLA only supports block_size 32 or 64
pytest.skip(
"FlashInfer MLA only supports block_size 32 or 64"
)
backend = get_attn_backend(
576, torch.float16, None, use_mla=use_mla
)
expected = "FLASHINFER_MLA"
assert backend.get_name() == expected
elif name == "FLASHMLA":
if block_size != 64:
# FlashMLA only supports block_size == 64
pytest.skip("FlashMLA only supports block_size 64")
from vllm.v1.attention.backends.mla.flashmla import (
is_flashmla_dense_supported,
)
is_supported, _ = is_flashmla_dense_supported()
if not is_supported:
pytest.skip("FlashMLA not supported on this platform")
backend = get_attn_backend(
576,
torch.float16,
None,
use_mla=use_mla,
)
expected = name
assert backend.get_name() == expected
elif name == "FLASH_ATTN_MLA":
from vllm.v1.attention.backends.fa_utils import (
flash_attn_supports_mla,
)
if not flash_attn_supports_mla():
pytest.skip(
"FlashAttention MLA not supported on this platform"
)
backend = get_attn_backend(
576, torch.float16, None, use_mla=use_mla
)
expected = "FLASH_ATTN_MLA"
assert backend.get_name() == expected
else:
# TRITON_MLA or other fallback
backend = get_attn_backend(
576, torch.float16, None, use_mla=use_mla
)
expected = "TRITON_MLA"
assert backend.get_name() == expected
elif name == "FLASHINFER":
backend = get_attn_backend(64, torch.float16, None, use_mla=use_mla)
expected = "FLASHINFER"
assert backend.get_name() == expected
elif name == "FLASH_ATTN":
backend = get_attn_backend(32, torch.float16, None, use_mla=use_mla)
expected = "FLASH_ATTN"
assert backend.get_name() == expected
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="GPU attention backends"
)
@pytest.mark.parametrize(
"backend_name",
[
"FLASH_ATTN",
"FLASH_ATTN_DIFFKV",
"FLASHINFER",
"TRITON_MLA",
"ROCM_AITER_MLA",
"ROCM_AITER_TRITON_MLA",
"CUTLASS_MLA",
"FLASHMLA",
"FLASHINFER_MLA",
"TOKENSPEED_MLA",
"FLASH_ATTN_MLA",
"FLASHMLA_SPARSE",
"FLASHINFER_MLA_SPARSE",
],
)
def test_supported_backend_preserves_dcp_eligibility(backend_name, default_vllm_config):
"""DCP opt-ins preserve eligibility and unrelated configuration restrictions."""
if backend_name.startswith("ROCM_") and not current_platform.is_rocm():
pytest.skip("ROCm-specific backend")
try:
backend_cls = AttentionBackendEnum[backend_name].get_class()
except ImportError as error:
pytest.skip(f"Optional backend dependency unavailable: {error}")
config = AttentionSelectorConfig(
head_size=576 if backend_cls.is_mla() else 128,
dtype=torch.bfloat16,
kv_cache_dtype="auto",
block_size=backend_cls.get_preferred_block_size(64),
use_mla=backend_cls.is_mla(),
use_sparse=backend_cls.is_sparse(),
)
kwargs = dict(
device_capability=current_platform.get_device_capability(), **config._asdict()
)
invalid_without_dcp = backend_cls.validate_configuration(**kwargs)
kwargs["use_dcp"] = True
assert backend_cls.validate_configuration(**kwargs) == invalid_without_dcp
@pytest.mark.parametrize("device", ["cpu", "cuda", "hip"])
def test_fp32_fallback(device: str):
"""Test attention backend selection with fp32."""
# Use default config (no backend specified)
vllm_config = VllmConfig()
with set_current_vllm_config(vllm_config):
if device == "cpu":
with patch("vllm.platforms.current_platform", CpuPlatform()):
backend = get_attn_backend(16, torch.float32, None)
assert backend.get_name() == "CPU_ATTN"
elif device == "cuda":
if CudaPlatform is None:
pytest.skip("CudaPlatform not available")
with patch("vllm.platforms.current_platform", CudaPlatform()):
backend = get_attn_backend(16, torch.float32, None)
assert backend.get_name() == "FLEX_ATTENTION"
elif device == "hip":
if RocmPlatform is None:
pytest.skip("RocmPlatform not available")
# ROCm backends do not support head_size=16 (minimum is 32).
# No known HuggingFace transformer model uses head_size=16.
# Revisit if a real model with this head size is identified
# and accuracy-tested.
with (
patch("vllm.platforms.current_platform", RocmPlatform()),
pytest.raises(ValueError, match="No valid attention backend"),
):
get_attn_backend(16, torch.float32, None)
def test_invalid_backend():
"""Test that invalid attention backend names raise ValueError."""
with (
pytest.raises(ValueError),
):
# Invalid backend name should raise ValueError when creating enum
AttentionConfig(backend=AttentionBackendEnum["INVALID"])
@pytest.mark.parametrize("auto_value", ["auto", "AUTO", "Auto"])
def test_auto_backend_string(auto_value: str):
"""Test that 'auto' string value triggers automatic backend selection."""
# Using "auto" should result in backend=None (automatic selection)
attention_config = AttentionConfig(backend=auto_value)
assert attention_config.backend is None
def test_auto_backend_selection_behavior():
"""Test that 'auto' backend behaves same as None (automatic selection)."""
# Create config with explicit "auto"
auto_config = AttentionConfig(backend="auto")
# Create config with None (default)
none_config = AttentionConfig(backend=None)
# Both should have backend=None
assert auto_config.backend is None
assert none_config.backend is None
# Both configs should result in the same automatic backend selection
vllm_config_auto = VllmConfig(attention_config=auto_config)
vllm_config_none = VllmConfig(attention_config=none_config)
with (
set_current_vllm_config(vllm_config_auto),
patch("vllm.platforms.current_platform", CpuPlatform()),
):
backend_auto = get_attn_backend(16, torch.float16, None)
_cached_get_attn_backend.cache_clear()
with (
set_current_vllm_config(vllm_config_none),
patch("vllm.platforms.current_platform", CpuPlatform()),
):
backend_none = get_attn_backend(16, torch.float16, None)
# Both should select the same backend
assert backend_auto.get_name() == backend_none.get_name()
@pytest.mark.parametrize(
"backend_name,flash_attn_version,should_succeed",
[
("FLASH_ATTN", 3, True), # FA3 supports per-head quant scales
("FLASH_ATTN", 2, False), # FA2 does not support per-head quant scales
("FLASHINFER", None, False), # FlashInfer does not support
("FLEX_ATTENTION", None, False), # Flex does not support
],
)
@pytest.mark.skipif(
current_platform.is_rocm(),
reason="Attention backend FA3 is not supported on ROCm. This test can't succeed.",
)
def test_per_head_quant_scales_backend_selection(
backend_name: str, flash_attn_version: Literal[2, 3, 4] | None, should_succeed: bool
):
"""Test backend selection when use_per_head_quant_scales=True."""
# Clear cache to ensure fresh backend selection
_cached_get_attn_backend.cache_clear()
attention_config = AttentionConfig(
backend=AttentionBackendEnum[backend_name],
flash_attn_version=flash_attn_version,
)
cache_config = CacheConfig(block_size=64)
vllm_config = VllmConfig(
attention_config=attention_config, cache_config=cache_config
)
if CudaPlatform is None:
pytest.skip("CudaPlatform not available")
with (
set_current_vllm_config(vllm_config),
patch("vllm.platforms.current_platform", CudaPlatform()),
):
if backend_name == "FLASH_ATTN" and flash_attn_version == 3:
if not torch.cuda.is_available():
pytest.skip("FA3 requires CUDA")
capability = torch.cuda.get_device_capability()
if capability[0] != 9:
pytest.skip("FA3 is only supported on Hopper (SM 9.x) GPUs")
if should_succeed:
backend = get_attn_backend(
head_size=128,
dtype=torch.float16,
kv_cache_dtype="fp8",
use_per_head_quant_scales=True,
)
assert backend.get_name() == backend_name
else:
with pytest.raises(ValueError) as exc_info:
get_attn_backend(
head_size=128,
dtype=torch.float16,
kv_cache_dtype="fp8",
use_per_head_quant_scales=True,
)
assert backend_name in str(exc_info.value)
@pytest.mark.skipif(
CudaPlatform is None, reason="CUDA platform is required for this test"
)
@pytest.mark.parametrize("head_size", [64, 80, 96, 128, 192, 256])
def test_int4_per_token_head_head_size_selection(head_size: int):
"""INT4 per-token-head KV cache is selectable at any even head size.
The Hadamard rotation the INT4 write path applies is now defined for
non-power-of-two rows as well (it runs block-diagonally, see
``fast_hadamard_transform``), so a model such as Phi-3 (head_size=96) must
keep selecting TRITON_ATTN instead of being rejected.
"""
vllm_config = VllmConfig(cache_config=CacheConfig(block_size=64))
with (
set_current_vllm_config(vllm_config),
patch("vllm.platforms.current_platform", CudaPlatform()),
):
backend = get_attn_backend(
head_size=head_size,
dtype=torch.float16,
kv_cache_dtype="int4_per_token_head",
)
assert backend.get_name() == "TRITON_ATTN"
@pytest.mark.parametrize(
"backend_name,use_non_causal,should_succeed",
[
("FLASH_ATTN", True, True), # FlashAttn supports non-causal
("FLASH_ATTN", False, True), # FlashAttn also works with causal
]
+ (
[
("FLASHINFER", True, True), # FlashInfer supports non-causal
("FLASHINFER", False, True), # FlashInfer works with causal
]
if CudaPlatform is not None
else []
),
)
def test_non_causal_backend_selection(
backend_name: str, use_non_causal: bool, should_succeed: bool
):
"""Test that use_non_causal on AttentionConfig controls backend filtering.
DFlashProposer sets use_non_causal=True on the draft model's
AttentionConfig so only non-causal-capable backends are selected.
The target model keeps use_non_causal=False (default) and can use
any backend.
"""
_cached_get_attn_backend.cache_clear()
attention_config = AttentionConfig(
backend=AttentionBackendEnum[backend_name],
use_non_causal=use_non_causal,
)
cache_config = CacheConfig(block_size=16)
vllm_config = VllmConfig(
attention_config=attention_config, cache_config=cache_config
)
platform = CudaPlatform or RocmPlatform
if platform is None:
pytest.skip("CudaPlatform and RocmPlatform are not available")
with (
set_current_vllm_config(vllm_config),
patch("vllm.platforms.current_platform", platform()),
):
if should_succeed:
backend = get_attn_backend(
head_size=128,
dtype=torch.float16,
kv_cache_dtype=None,
)
assert backend.get_name() == backend_name
else:
with pytest.raises(ValueError) as exc_info:
get_attn_backend(
head_size=128,
dtype=torch.float16,
kv_cache_dtype=None,
)
assert "non-causal" in str(exc_info.value).lower()
def test_non_causal_autoselect_backend():
"""Test that when backend=None with use_non_causal=True, auto-selection
picks a compatible backend.
This simulates the DFlash scenario where the user doesn't specify
--attention-backend or --speculative-config.attention_backend.
The drafter inherits backend=None and auto-selects a backend that
supports non-causal attention.
"""
_cached_get_attn_backend.cache_clear()
attention_config = AttentionConfig(
backend=None,
use_non_causal=True,
)
cache_config = CacheConfig(block_size=16)
vllm_config = VllmConfig(
attention_config=attention_config, cache_config=cache_config
)
if CudaPlatform is None:
pytest.skip("CudaPlatform not available")
with (
set_current_vllm_config(vllm_config),
patch("vllm.platforms.current_platform", CudaPlatform()),
):
backend = get_attn_backend(
head_size=128,
dtype=torch.float16,
kv_cache_dtype=None,
)
assert backend.supports_non_causal()
@pytest.mark.parametrize(
"kv_cache_dtype",
[
"fp8_e5m2",
"fp8_ds_mla",
"fp8_inc",
"nvfp4",
"nvfp4_4over6",
"fp8_per_token_head",
"int8_per_token_head",
],
)
def test_flash_attn_rejects_unhandled_kv_cache_dtypes(kv_cache_dtype: CacheDType):
"""FlashAttentionBackend must not claim support for kv_cache dtypes
that it cannot handle."""
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
assert not FlashAttentionBackend.supports_kv_cache_dtype(kv_cache_dtype)
@pytest.mark.parametrize("kv_cache_dtype", ["fp8", "fp8_e4m3"])
def test_flash_attn_accepts_handled_fp8_variants(
kv_cache_dtype: CacheDType, monkeypatch: pytest.MonkeyPatch
):
"""FlashAttentionBackend must accept the two fp8 dtypes it can actually
handle: 'fp8' (alias for fp8_e4m3fn) and 'fp8_e4m3'."""
import vllm.v1.attention.backends.fa_utils as fa_utils_mod
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
# The fp8 decision is made in fa_utils, using its own current_platform
# binding, so patch is_xpu there (not on flash_attn's) to stay robust to
# import order across earlier tests that patch vllm.platforms.current_platform.
monkeypatch.setattr(fa_utils_mod.current_platform, "is_xpu", lambda: True)
assert FlashAttentionBackend.supports_kv_cache_dtype(kv_cache_dtype)
blackwell_only = pytest.mark.skipif(
not current_platform.is_cuda(), reason="FA4 is CUDA-only"
)
@contextmanager
def _blackwell(vllm_config=None):
platform = MagicMock()
platform.is_xpu.return_value = False
platform.is_rocm.return_value = False
platform.get_device_capability.return_value = DeviceCapability(10, 0)
with (
patch("vllm.v1.attention.backends.fa_utils.current_platform", platform),
patch(
"vllm.vllm_flash_attn.flash_attn_interface.is_fa_version_supported",
return_value=True,
),
patch("vllm.config.get_current_vllm_config_or_none", return_value=vllm_config),
patch(
"vllm.v1.attention.backends.flash_attn.get_current_vllm_config_or_none",
return_value=vllm_config,
),
):
yield
def _hd256_config(
*,
is_mm_prefix_lm=False,
rswa_window=None,
dcp_size=1,
softcap=None,
head_size=256,
cache_dtype="auto",
):
vllm_config = MagicMock()
vllm_config.attention_config.flash_attn_version = None
vllm_config.model_config.is_mm_prefix_lm = is_mm_prefix_lm
vllm_config.model_config.rswa_window = rswa_window
vllm_config.model_config.hf_text_config.attn_logit_softcapping = softcap
vllm_config.model_config.get_head_size.return_value = head_size
vllm_config.cache_config.cache_dtype = cache_dtype
vllm_config.parallel_config.decode_context_parallel_size = dcp_size
return vllm_config
@blackwell_only
@pytest.mark.parametrize(
"kwargs,config_kwargs,expected",
[
({}, {}, 4),
({"supports_fa4_hd256": False}, {}, 2),
({"has_sinks": True}, {}, 2),
({"requires_softcap": True}, {}, 2),
# Larger block sizes normalize to 128.
({"kv_cache_block_size": 16}, {}, 2),
({"kv_cache_block_size": 64}, {}, 2),
({"kv_cache_block_size": 256}, {}, 4),
({}, {"is_mm_prefix_lm": True}, 2),
({}, {"rswa_window": 512}, 2),
({}, {"dcp_size": 2}, 2),
({}, {"softcap": 50.0}, 2),
({}, {"cache_dtype": "fp8"}, 2),
({"head_size": 128, "kv_cache_block_size": 16}, {}, 4),
({"head_size": 192, "head_size_v": 128, "kv_cache_block_size": 16}, {}, 4),
({"head_size": 256, "head_size_v": 128, "kv_cache_block_size": 16}, {}, 2),
({"head_size": 256, "head_size_v": 64}, {}, 2),
],
)
def test_fa4_hd256_fallback_matrix(kwargs, config_kwargs, expected):
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
kwargs = {
"head_size": 256,
"kv_cache_block_size": 128,
"supports_fa4_hd256": True,
**kwargs,
}
with _blackwell(_hd256_config(head_size=kwargs["head_size"], **config_kwargs)):
assert get_flash_attn_version(**kwargs) == expected
@blackwell_only
@pytest.mark.parametrize(
"backend_name,config_kwargs,expected",
[
("FLASH_ATTN", {}, 128),
("FLASH_ATTN", {"softcap": 50.0}, 16),
("FLASH_ATTN", {"head_size": 128}, 16),
("FLASH_ATTN", {"cache_dtype": "fp8"}, 16),
("FLASH_ATTN_DIFFKV", {}, 16),
],
)
def test_fa4_hd256_block_size_advertisement(
backend_name: str, config_kwargs: dict, expected: int
):
from vllm.v1.attention.backend import MultipleOf
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
from vllm.v1.attention.backends.flash_attn_diffkv import (
FlashAttentionDiffKVBackend,
)
backend = {
"FLASH_ATTN": FlashAttentionBackend,
"FLASH_ATTN_DIFFKV": FlashAttentionDiffKVBackend,
}[backend_name]
vllm_config = _hd256_config(**config_kwargs)
with _blackwell(vllm_config):
(size,) = backend.get_supported_kernel_block_sizes()
preferred = backend.get_preferred_block_size(16)
assert preferred == expected
if expected == 128:
assert size == 128
else:
assert isinstance(size, MultipleOf) and size.base == expected
@blackwell_only
def test_fa4_hd256_mm_prefix_deselects_flash_attn():
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
with _blackwell(_hd256_config(is_mm_prefix_lm=True)):
assert (
FlashAttentionBackend.supports_combination(
head_size=256,
dtype=torch.bfloat16,
kv_cache_dtype=None,
block_size=128,
use_mla=False,
has_sink=False,
use_sparse=False,
use_mm_prefix=True,
device_capability=DeviceCapability(10, 0),
)
is not None
)
@pytest.mark.skipif(
not current_platform.is_cuda()
or not current_platform.is_device_capability_family(100),
reason="requires a Blackwell GPU",
)
@pytest.mark.parametrize(
"attn_type,sliding_window,expected",
[
(AttentionType.DECODER, None, 4),
(AttentionType.DECODER, 512, 4),
# Local encoder attention lacks the required seqused tensors.
(AttentionType.ENCODER_ONLY, None, 4),
(AttentionType.ENCODER_ONLY, 512, 2),
],
)
def test_fa4_hd256_impl_selection(attn_type, sliding_window, expected):
from vllm.v1.attention.backends.flash_attn import FlashAttentionImpl
# Implementations are constructed before the platform updates the block size.
with set_current_vllm_config(VllmConfig()):
impl = FlashAttentionImpl(
num_heads=8,
head_size=256,
scale=0.0625,
num_kv_heads=8,
alibi_slopes=None,
sliding_window=sliding_window,
kv_cache_dtype="auto",
attn_type=attn_type,
)
assert impl.vllm_flash_attn_version == expected
assert impl.fa4_hd256 == (expected == 4)
@pytest.fixture
def blackwell_selection():
with (
patch.object(
type(current_platform),
"get_device_capability",
return_value=DeviceCapability(10, 0),
),
patch(
"vllm.v1.attention.backends.fa_utils.is_fa_version_supported",
return_value=True,
),
):
yield
@pytest.fixture
def hopper_selection():
with (
patch.object(
type(current_platform),
"get_device_capability",
return_value=DeviceCapability(9, 0),
),
patch(
"vllm.v1.attention.backends.fa_utils.is_fa_version_supported",
return_value=True,
),
):
yield
@blackwell_only
@pytest.mark.parametrize("use_mm_prefix", [False, True])
@pytest.mark.parametrize("flash_attn_version", [None, 3, 4])
def test_hopper_mm_prefix_selects_triton_flash_attn(
use_mm_prefix, flash_attn_version, hopper_selection
):
"""Hopper selects the composite when its causal route resolves FA3 or FA4."""
from vllm.engine.arg_utils import EngineArgs
config = EngineArgs(
model="google/gemma-4-31B-it",
dtype="bfloat16",
attention_config=AttentionConfig(flash_attn_version=flash_attn_version),
).create_engine_config()
with set_current_vllm_config(config):
backend = get_attn_backend(
256, torch.bfloat16, None, use_mm_prefix=use_mm_prefix
)
expected = "TRITON_FLASH_ATTN" if use_mm_prefix else "FLASH_ATTN"
assert backend.get_name() == expected
@blackwell_only
@pytest.mark.parametrize("use_mm_prefix", [False, True])
@pytest.mark.parametrize("kv_cache_dtype", [None, "fp8_e4m3"])
def test_mm_prefix_selects_composite_without_changing_causal_default(
use_mm_prefix, kv_cache_dtype, blackwell_selection
):
"""The image-mask requirement must reach CUDA's automatic backend priority."""
from vllm.engine.arg_utils import EngineArgs
config = EngineArgs(
model="google/gemma-4-31B-it", dtype="bfloat16"
).create_engine_config()
with set_current_vllm_config(config):
backend = get_attn_backend(
256, torch.bfloat16, kv_cache_dtype, use_mm_prefix=use_mm_prefix
)
expected = "TRITON_FLASHINFER" if use_mm_prefix else "FLASHINFER"
assert backend.get_name() == expected
@blackwell_only
@pytest.mark.parametrize(
"feature_kwargs,reason",
[
({"use_rswa": True}, "R-SWA not supported"),
({"kv_cache_dtype": "int8_per_token_head"}, "kv_cache_dtype not supported"),
({"block_size": 16}, "block_size not supported"),
({"use_dcp": True}, "DCP not supported"),
({"use_pcp": True}, "PCP not supported"),
({"use_adaptive_verification": True}, "device-cpu query lens mismatch"),
],
)
def test_composite_rejects_features_not_shared_by_both_routes(feature_kwargs, reason):
"""Automatic selection must not admit features only one child can execute."""
from vllm.engine.arg_utils import EngineArgs
from vllm.v1.attention.backends.triton_flashinfer import TritonFlashInferBackend
config = EngineArgs(
model="google/gemma-4-31B-it", dtype="bfloat16"
).create_engine_config()
args = dict(
head_size=256,
dtype=torch.bfloat16,
kv_cache_dtype="auto",
block_size=128,
use_mla=False,
has_sink=False,
use_sparse=False,
use_mm_prefix=True,
use_per_head_quant_scales=False,
device_capability=DeviceCapability(10, 0),
attn_type=AttentionType.DECODER,
)
args.update(feature_kwargs)
with set_current_vllm_config(config):
reasons = TritonFlashInferBackend.validate_configuration(**args)
assert any(reason in message for message in reasons), reasons
@blackwell_only
def test_rswa_selection_does_not_reuse_causal_result(blackwell_selection):
"""An R-SWA requirement must invalidate a previously cached causal selection."""
from vllm.engine.arg_utils import EngineArgs
config = EngineArgs(
model="google/gemma-4-31B-it",
dtype="bfloat16",
attention_config=AttentionConfig(
backend=AttentionBackendEnum.TRITON_FLASHINFER
),
).create_engine_config()
with set_current_vllm_config(config):
assert (
get_attn_backend(256, torch.bfloat16, None).get_name()
== "TRITON_FLASHINFER"
)
config.model_config.model_arch_config.rswa_window = 128
with pytest.raises(ValueError, match="R-SWA"):
get_attn_backend(256, torch.bfloat16, None)
config.attention_config.backend = AttentionBackendEnum.TRITON_ATTN
assert get_attn_backend(256, torch.bfloat16, None).get_name() == "TRITON_ATTN"
File diff suppressed because it is too large Load Diff
@@ -1,410 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import random
import pytest
import torch
from vllm.model_executor.layers.sparse_attn_indexer import as_fp32_logits
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import (
_ceil_to_ue8m0,
calc_diff,
fp8_fp4_mqa_logits,
fp8_fp4_paged_mqa_logits,
get_num_sms,
get_paged_mqa_logits_metadata,
mqa_logits_mx_inputs,
mqa_logits_use_mx_scales,
native_next_n_supported,
)
from vllm.utils.import_utils import has_deep_gemm
from vllm.utils.math_utils import cdiv
def encode_indexer_scales(sf: torch.Tensor) -> torch.Tensor:
"""Per-token power-of-two fp32 scales as the indexer K cache stores them:
four equal packed UE8M0 bytes under MX scales, else the fp32 itself."""
if not mqa_logits_use_mx_scales():
return sf
ue8m0 = (sf.view(torch.int32) >> 23).to(torch.uint8)
return ue8m0[..., None].repeat_interleave(4, dim=-1).view(torch.int32)[..., 0]
def kv_cache_cast_to_fp8(x: torch.Tensor) -> torch.Tensor:
# x: (num_blocks, block_size, 1, head_dim)
num_blocks, block_size, num_heads, head_dim = x.shape
assert num_heads == 1
x_amax = x.abs().float().amax(dim=3, keepdim=True).clamp(1e-4)
sf = _ceil_to_ue8m0(x_amax / 448.0)
x_scaled = (x * (1.0 / sf)).to(torch.float8_e4m3fn)
x_fp8 = torch.empty(
(num_blocks, block_size * (head_dim + 4)),
device=x.device,
dtype=torch.uint8,
)
x_fp8[:, : block_size * head_dim] = x_scaled.view(
num_blocks, block_size * head_dim
).view(dtype=torch.uint8)
x_fp8[:, block_size * head_dim :] = encode_indexer_scales(
sf.view(num_blocks, block_size)
).view(dtype=torch.uint8)
return x_fp8.view(num_blocks, block_size, num_heads, head_dim + 4)
def per_custom_dims_cast_to_fp8(
x: torch.Tensor, dims: tuple, use_ue8m0: bool
) -> tuple[torch.Tensor, torch.Tensor]:
excluded_dims = tuple([i for i in range(x.dim()) if i not in set(dims)])
x_amax = x.abs().float().amax(dim=excluded_dims, keepdim=True).clamp(1e-4)
sf = x_amax / 448.0
sf = _ceil_to_ue8m0(sf) if use_ue8m0 else sf
x_scaled = (x * (1.0 / sf)).to(torch.float8_e4m3fn)
return x_scaled, sf.squeeze()
def _generate_cp_test_data(seq_len: int, seq_len_kv: int):
assert seq_len_kv % seq_len == 0 and seq_len % 2 == 0
chunk_size = seq_len // 2
cp_size = seq_len_kv // seq_len
cp_id = cp_size // 3
ks = torch.zeros(seq_len, dtype=torch.int, device="cuda")
ke = torch.zeros(seq_len, dtype=torch.int, device="cuda")
for i in range(chunk_size):
ke[i] = cp_id * chunk_size + i
ke[i + chunk_size] = (cp_size * 2 - 1 - cp_id) * chunk_size + i
return ks, ke
def _ref_fp8_mqa_logits(
q: torch.Tensor,
kv: torch.Tensor,
weights: torch.Tensor,
cu_seqlen_ks: torch.Tensor,
cu_seqlen_ke: torch.Tensor,
):
seq_len_kv = kv.shape[0]
k = kv
q = q.float()
k = k.float()
mask_lo = (
torch.arange(0, seq_len_kv, device="cuda")[None, :] >= cu_seqlen_ks[:, None]
)
mask_hi = (
torch.arange(0, seq_len_kv, device="cuda")[None, :] < cu_seqlen_ke[:, None]
)
mask = mask_lo & mask_hi
score = torch.einsum("mhd,nd->hmn", q, k)
logits = (score.relu() * weights.unsqueeze(-1).transpose(0, 1)).sum(dim=0)
logits = logits.masked_fill(~mask, float("-inf"))
return logits
def _packed_requests_test_data(seq_len: int, seq_len_kv: int):
# Two causal prefill requests packed into one KV buffer, so the second
# request's rows start at a nonzero K position.
half, half_kv = seq_len // 2, seq_len_kv // 2
rows = torch.arange(seq_len, dtype=torch.int, device="cuda")
ks = torch.where(rows < half, 0, half_kv).to(torch.int)
ke = ks + half_kv - half + 1 + rows % half
return ks, ke
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
@pytest.mark.skipif(not has_deep_gemm(), reason="DeepGEMM not available")
@pytest.mark.skipif(
not current_platform.has_device_capability(90), reason="SM90 and SM100 only"
)
@pytest.mark.parametrize("layout", ["cp", "no_cp", "packed"])
def test_deepgemm_fp8_mqa_logits(layout: str):
"""FP8 indexer prefill logits, compressed to each row's valid K span."""
torch.manual_seed(0)
random.seed(0)
num_heads, head_dim = 32, 128
seq_len, seq_len_kv = 512, 1024
q = torch.randn(seq_len, num_heads, head_dim, device="cuda", dtype=torch.bfloat16)
kv = torch.randn(seq_len_kv, head_dim, device="cuda", dtype=torch.bfloat16)
weights = torch.randn(seq_len, num_heads, device="cuda", dtype=torch.float32)
if layout == "cp":
ks, ke = _generate_cp_test_data(seq_len, seq_len_kv)
elif layout == "no_cp":
ks = torch.zeros(seq_len, dtype=torch.int, device="cuda")
ke = torch.arange(seq_len, dtype=torch.int, device="cuda") + (
seq_len_kv - seq_len
)
else:
ks, ke = _packed_requests_test_data(seq_len, seq_len_kv)
q_fp8 = q.to(torch.float8_e4m3fn)
kv_fp8, kv_sf = per_custom_dims_cast_to_fp8(kv, (0,), True)
q_scale, kernel_weights = mqa_logits_mx_inputs(q_fp8, None, weights)
max_seqlen_k = int((ke - ks).max())
logits = fp8_fp4_mqa_logits(
(q_fp8, q_scale),
(kv_fp8, encode_indexer_scales(kv_sf)),
kernel_weights,
ks,
ke,
max_seqlen_k,
)
assert logits.shape == (seq_len, max_seqlen_k)
logits = as_fp32_logits(logits, ke - ks)
ref_logits = _ref_fp8_mqa_logits(
q=q,
kv=kv,
weights=kernel_weights.float(),
cu_seqlen_ks=ks,
cu_seqlen_ke=ke,
)
# Compressed row i holds K positions [ks[i], ke[i]) from column zero.
cols = ks[:, None] + torch.arange(max_seqlen_k, device="cuda")[None, :]
valid = cols < ke[:, None]
ref_logits = ref_logits.gather(1, cols.clamp(max=seq_len_kv - 1).long())
ref_logits = ref_logits.masked_fill(~valid, 0)
logits = logits.masked_fill(~valid, 0)
diff = calc_diff(logits, ref_logits)
assert diff < 1e-3, f"{diff=}"
def _ref_fp8_fp4_paged_mqa_logits(
q: torch.Tensor,
kv_cache: torch.Tensor,
weights: torch.Tensor,
context_lens: torch.Tensor,
block_tables: torch.Tensor,
max_model_len: int,
):
batch_size, next_n, _, _ = q.size()
_, block_size, _, _ = kv_cache.size()
logits = torch.full(
[batch_size * next_n, max_model_len],
float("-inf"),
device=q.device,
dtype=torch.float32,
)
context_lens_list = context_lens.tolist()
for i in range(batch_size):
context_len = context_lens_list[i]
q_offsets = torch.arange(context_len - next_n, context_len, device="cuda")
weight_slice = (
weights[i * next_n : (i + 1) * next_n, :].transpose(0, 1).contiguous()
)
for block_rk in range(cdiv(context_len, block_size)):
block_idx = block_tables[i][block_rk]
qx, kx = q[i], kv_cache[block_idx]
k_offsets = torch.arange(
block_rk * block_size,
(block_rk + 1) * block_size,
device="cuda",
)
mask = (k_offsets[None, :] < context_len) & (
k_offsets[None, :] <= q_offsets[:, None]
)
s = torch.where(
mask[None, :, :],
(qx.transpose(0, 1) @ kx.transpose(0, 1).transpose(1, 2)).to(
logits.dtype
),
float("-inf"),
)
s = torch.relu(s) * weight_slice[..., None]
s = s.sum(dim=0)
logits[
i * next_n : (i + 1) * next_n,
block_rk * block_size : (block_rk + 1) * block_size,
] = torch.where(k_offsets[None, :] <= q_offsets[:, None], s, float("-inf"))
return logits
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
@pytest.mark.skipif(not has_deep_gemm(), reason="DeepGEMM not available")
@pytest.mark.skipif(
not current_platform.has_device_capability(90), reason="SM90 and SM100 only"
)
# next_n = 1 + num_speculative_tokens, so next_n=4 is MTP=3 (issue #35878).
@pytest.mark.parametrize("batch_size,next_n", [(4, 1), (2, 2), (2, 4)])
def test_deepgemm_fp8_fp4_paged_mqa_logits(batch_size: int, next_n: int):
if not native_next_n_supported(next_n):
pytest.skip(f"next_n={next_n} has no native kernel on this architecture")
torch.manual_seed(0)
random.seed(0)
max_model_len = 4096
for heads, index_dim in [(32, 128)]:
for avg_kv in (2048,):
num_blocks, blocksize = max_model_len * 2, 64
q = torch.randn(
(batch_size, next_n, heads, index_dim),
device="cuda",
dtype=torch.bfloat16,
)
kv_cache = torch.randn(
(num_blocks, blocksize, 1, index_dim),
device="cuda",
dtype=torch.bfloat16,
)
weights = torch.randn(
(batch_size * next_n, heads),
device="cuda",
dtype=torch.float32,
)
context_lens = (
torch.randint(int(0.8 * avg_kv), int(1.2 * avg_kv), (batch_size,))
.cuda()
.to(torch.int32)
)
max_block_len = (
(context_lens.max().item() + blocksize - 1) // blocksize * blocksize
)
block_tables = torch.zeros(
(batch_size, max_block_len),
device="cuda",
dtype=torch.int32,
)
counter = 0
block_idx_pool = list(range(num_blocks))
random.shuffle(block_idx_pool)
for i in range(batch_size):
ctx_len = int(context_lens[i].item())
for j in range((ctx_len + blocksize - 1) // blocksize):
block_tables[i][j] = block_idx_pool[counter]
counter += 1
q_fp8 = q.to(torch.float8_e4m3fn)
kv_cache_fp8 = kv_cache_cast_to_fp8(kv_cache)
# deep_gemm paged MQA logits requires 2D context_lens of
# shape (B, next_n) (csrc/apis/attention.hpp:332-335);
# see indexer.py:607-608. For each batch/next_n token, the
# effective context length is context_lens[b] - next_n + j + 1.
next_n_arange = torch.arange(next_n, device="cuda", dtype=torch.int32)
context_lens_2d = (
context_lens.unsqueeze(-1) - next_n + 1 + next_n_arange
).contiguous()
q_scale, kernel_weights = mqa_logits_mx_inputs(q_fp8, None, weights)
kernel_q = (q_fp8, q_scale)
kernel_context_lens, kernel_block_tables = context_lens_2d, block_tables
indices = None
if mqa_logits_use_mx_scales():
# SM100 only takes varlen rows: one Q token each, tied to its
# request by `indices`.
assert q_scale is not None
kernel_q = (
q_fp8.flatten(0, 1)[:, None],
q_scale.flatten(0, 1)[:, None],
)
kernel_context_lens = context_lens_2d.view(-1, 1)
kernel_block_tables = block_tables.repeat_interleave(next_n, dim=0)
indices = torch.arange(batch_size, device="cuda", dtype=torch.int32)
indices = indices.repeat_interleave(next_n)
schedule_metadata = get_paged_mqa_logits_metadata(
kernel_context_lens, blocksize, get_num_sms(), indices=indices
)
logits = fp8_fp4_paged_mqa_logits(
kernel_q,
kv_cache_fp8,
kernel_weights,
kernel_context_lens,
kernel_block_tables,
schedule_metadata,
max_model_len,
indices=indices,
)
if indices is not None:
logits = as_fp32_logits(logits, kernel_context_lens.view(-1))
ref_logits = _ref_fp8_fp4_paged_mqa_logits(
q,
kv_cache,
kernel_weights.float(),
context_lens,
block_tables,
max_model_len,
)
positions = (
torch.arange(max_model_len, device="cuda")
.unsqueeze(0)
.expand(batch_size * next_n, -1)
)
row_indices = torch.arange(batch_size * next_n, device="cuda") // next_n
next_n_offset = torch.arange(batch_size * next_n, device="cuda") % next_n
mask = positions <= (
context_lens[row_indices] - next_n + next_n_offset
).unsqueeze(1)
logits = logits.float().masked_fill(~mask, 0)
ref_logits = ref_logits.masked_fill(~mask, 0)
diff = calc_diff(logits, ref_logits)
assert diff < 1e-3, f"{diff=}"
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
@pytest.mark.skipif(not has_deep_gemm(), reason="DeepGEMM not available")
@pytest.mark.skipif(
not current_platform.has_device_capability(90), reason="SM90 and SM100 only"
)
def test_deepgemm_paged_mqa_packed_manager_block_stride():
torch.manual_seed(0)
num_blocks, block_size, packed_rows = 4, 256, 320
num_heads, head_dim, row_bytes = 64, 128, 132
page_size, pages_per_block = 64, block_size // 64
dense_stride, packed_stride = pages_per_block, packed_rows // page_size
compact_pages = kv_cache_cast_to_fp8(
torch.randn(
num_blocks, block_size, 1, head_dim, device="cuda", dtype=torch.bfloat16
).view(num_blocks * pages_per_block, page_size, 1, head_dim)
)
packed_backing = torch.zeros(
num_blocks,
packed_rows,
row_bytes,
device="cuda",
dtype=torch.uint8,
)
packed_manager = packed_backing[:, :block_size]
packed_manager.copy_(compact_pages.squeeze(2).view(num_blocks, block_size, -1))
packed_pages = packed_backing.view(-1, page_size, row_bytes).unsqueeze(2)
offsets = torch.arange(pages_per_block, device="cuda", dtype=torch.int32)
tables = [
(stride + offsets).unsqueeze(0) for stride in (dense_stride, packed_stride)
]
q = torch.randn(1, 1, num_heads, head_dim, device="cuda", dtype=torch.bfloat16).to(
torch.float8_e4m3fn
)
q_scale, weights = mqa_logits_mx_inputs(
q, None, torch.randn(1, num_heads, device="cuda", dtype=torch.float32)
)
context_lens = torch.full((1, 1), block_size, device="cuda", dtype=torch.int32)
indices = (
torch.zeros(1, device="cuda", dtype=torch.int32)
if mqa_logits_use_mx_scales()
else None
)
schedule = get_paged_mqa_logits_metadata(
context_lens, page_size, get_num_sms(), indices=indices
)
logits = [
fp8_fp4_paged_mqa_logits(
(q, q_scale),
cache,
weights,
context_lens,
table,
schedule,
block_size,
indices=indices,
)
for cache, table in zip((compact_pages, packed_pages), tables)
]
torch.testing.assert_close(*logits, rtol=0, atol=0)
-737
View File
@@ -1,737 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.kv_cache_layout import KVCacheLayout
try:
import flashinfer
except ImportError:
if current_platform.is_rocm():
pytest.skip(
"flashinfer is not supported for vLLM on ROCm.", allow_module_level=True
)
import torch
NUM_HEADS = [(32, 8), (6, 1)]
HEAD_SIZES = [128, 256]
BLOCK_SIZES = [16, 32]
DTYPES = [torch.bfloat16]
NUM_BLOCKS = 32768 # Large enough to test overflow in index calculation.
SOFT_CAPS = [None, 30.0]
SLIDING_WINDOWS = [None, 64]
def ref_paged_attn(
query: torch.Tensor,
key_cache: torch.Tensor,
value_cache: torch.Tensor,
query_lens: list[int],
kv_lens: list[int],
block_tables: torch.Tensor,
scale: float,
sliding_window: int | None = None,
soft_cap: float | None = None,
) -> torch.Tensor:
num_seqs = len(query_lens)
block_tables = block_tables.cpu().numpy()
_, block_size, num_kv_heads, head_size = key_cache.shape
outputs: list[torch.Tensor] = []
start_idx = 0
for i in range(num_seqs):
query_len = query_lens[i]
kv_len = kv_lens[i]
q = query[start_idx : start_idx + query_len]
q *= scale
num_kv_blocks = (kv_len + block_size - 1) // block_size
block_indices = block_tables[i, :num_kv_blocks]
k = key_cache[block_indices].view(-1, num_kv_heads, head_size)
k = k[:kv_len]
v = value_cache[block_indices].view(-1, num_kv_heads, head_size)
v = v[:kv_len]
if q.shape[1] != k.shape[1]:
k = torch.repeat_interleave(k, q.shape[1] // k.shape[1], dim=1)
v = torch.repeat_interleave(v, q.shape[1] // v.shape[1], dim=1)
attn = torch.einsum("qhd,khd->hqk", q, k).float()
empty_mask = torch.ones(query_len, kv_len)
mask = torch.triu(empty_mask, diagonal=kv_len - query_len + 1).bool()
if sliding_window is not None:
sliding_window_mask = (
torch.triu(
empty_mask, diagonal=kv_len - (query_len + sliding_window) + 1
)
.bool()
.logical_not()
)
mask |= sliding_window_mask
if soft_cap is not None:
attn = soft_cap * torch.tanh(attn / soft_cap)
attn.masked_fill_(mask, float("-inf"))
attn = torch.softmax(attn, dim=-1).to(v.dtype)
out = torch.einsum("hqk,khd->qhd", attn, v)
outputs.append(out)
start_idx += query_len
return torch.cat(outputs, dim=0)
def _make_paged_kv_metadata(
kv_lens: list[int],
block_size: int,
num_blocks: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Build paged-KV metadata tensors for fast_plan_decode tests.
Returns:
kv_indptr – CPU int32, shape [num_seqs + 1]
kv_indices – CUDA int32, shape [total_blocks]
kv_last_page_lens – CPU int32, shape [num_seqs]
block_tables – CUDA int32, shape [num_seqs, max_blocks_per_seq]
"""
num_seqs = len(kv_lens)
max_blocks = (max(kv_lens) + block_size - 1) // block_size
block_tables = torch.randint(
0, num_blocks, (num_seqs, max_blocks), dtype=torch.int32, device="cuda"
)
indptr_list = [0]
indices_list: list[int] = []
last_lens_list: list[int] = []
for i, seq_len in enumerate(kv_lens):
n = (seq_len + block_size - 1) // block_size
indices_list.extend(block_tables[i, :n].cpu().tolist())
indptr_list.append(indptr_list[-1] + n)
last_lens_list.append(seq_len % block_size or block_size)
return (
torch.tensor(indptr_list, dtype=torch.int32, device="cpu"),
torch.tensor(indices_list, dtype=torch.int32, device="cuda"),
torch.tensor(last_lens_list, dtype=torch.int32, device="cpu"),
block_tables,
)
def _make_cg_decode_wrapper(
num_seqs: int,
kv_indices_buffer: torch.Tensor,
workspace_buffer: torch.Tensor,
use_tensor_cores: bool = True,
) -> "flashinfer.BatchDecodeWithPagedKVCacheWrapper":
"""Create a cudagraph-enabled BatchDecodeWithPagedKVCacheWrapper.
*kv_indices_buffer* is shared with the caller so that fast_plan_decode
can avoid the device-to-device index copy on subsequent (cudagraph) calls.
"""
return flashinfer.BatchDecodeWithPagedKVCacheWrapper(
workspace_buffer,
"NHD",
use_cuda_graph=True,
paged_kv_indptr_buffer=torch.zeros(
num_seqs + 1, dtype=torch.int32, device="cuda"
),
paged_kv_indices_buffer=kv_indices_buffer,
paged_kv_last_page_len_buffer=torch.zeros(
num_seqs, dtype=torch.int32, device="cuda"
),
use_tensor_cores=use_tensor_cores,
)
def test_fast_decode_plan_importable() -> None:
"""fast_decode_plan must be importable from flashinfer.decode.
This is a forward-compatibility smoke test: if FlashInfer reorganises its
public API the import will fail before any other test does.
"""
from flashinfer.decode import fast_decode_plan # noqa: F401
assert callable(fast_decode_plan)
@pytest.mark.parametrize("dtype", DTYPES)
@torch.inference_mode
def test_fast_plan_decode_warmup_uses_full_plan(dtype: torch.dtype) -> None:
"""On the first call fast_plan_decode must route through self.plan() and
flip vllm_first_call to False on the wrapper object."""
from unittest.mock import patch
from vllm.v1.attention.backends.flashinfer import fast_plan_decode
torch.set_default_device("cuda")
set_random_seed(0)
kv_lens = [128, 64]
block_size = 16
num_seqs = len(kv_lens)
num_query_heads, num_kv_heads = 8, 2
head_size = 128
kv_indptr, kv_indices, kv_last_page_lens, _ = _make_paged_kv_metadata(
kv_lens, block_size, NUM_BLOCKS
)
workspace = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
wrapper = _make_cg_decode_wrapper(num_seqs, kv_indices.clone(), workspace)
assert getattr(wrapper, "vllm_first_call", True) is True
with patch.object(wrapper, "plan", wraps=wrapper.plan) as mock_plan:
fast_plan_decode(
wrapper,
indptr_cpu=kv_indptr,
indices=kv_indices,
last_page_len_cpu=kv_last_page_lens,
seq_lens_cpu=torch.tensor(kv_lens, dtype=torch.int32, device="cpu"),
num_qo_heads=num_query_heads,
num_kv_heads=num_kv_heads,
head_dim=head_size,
page_size=block_size,
q_data_type=dtype,
kv_data_type=dtype,
)
mock_plan.assert_called_once()
assert wrapper.vllm_first_call is False, (
"vllm_first_call should be False after the first fast_plan_decode call"
)
@pytest.mark.parametrize("kv_lens", [[1328, 18, 463], [1, 54, 293, 70]])
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("head_size", HEAD_SIZES)
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
@pytest.mark.parametrize("dtype", DTYPES)
@torch.inference_mode
def test_fast_plan_decode_matches_full_plan(
kv_lens: list[int],
num_heads: tuple[int, int],
head_size: int,
block_size: int,
dtype: torch.dtype,
) -> None:
"""fast_plan_decode's cudagraph path (delegating to FlashInfer's
fast_decode_plan) must produce attention output numerically identical to
a standard plan() call.
Both the warmup call (self.plan) and the subsequent fast call
(fast_decode_plan) are verified against the same reference.
"""
from vllm.v1.attention.backends.flashinfer import fast_plan_decode
torch.set_default_device("cuda")
set_random_seed(0)
num_seqs = len(kv_lens)
num_query_heads, num_kv_heads = num_heads
query = torch.randn(num_seqs, num_query_heads, head_size, dtype=dtype)
key_value_cache = torch.randn(
NUM_BLOCKS, 2, block_size, num_kv_heads, head_size, dtype=dtype
)
kv_indptr, kv_indices, kv_last_page_lens, _ = _make_paged_kv_metadata(
kv_lens, block_size, NUM_BLOCKS
)
# Reference output via the standard plan()
workspace_ref = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
ref_wrapper = flashinfer.BatchDecodeWithPagedKVCacheWrapper(
workspace_ref, "NHD", use_tensor_cores=True
)
ref_wrapper.plan(
kv_indptr,
kv_indices,
kv_last_page_lens,
num_query_heads,
num_kv_heads,
head_size,
block_size,
"NONE",
q_data_type=dtype,
kv_data_type=dtype,
)
ref_output = ref_wrapper.run(query, key_value_cache)
# CUDAGraph wrapper exercised through fast_plan_decode
kv_indices_buf = kv_indices.clone()
workspace_cg = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
cg_wrapper = _make_cg_decode_wrapper(num_seqs, kv_indices_buf, workspace_cg)
plan_kwargs: dict = dict(
indptr_cpu=kv_indptr,
indices=kv_indices_buf,
last_page_len_cpu=kv_last_page_lens,
seq_lens_cpu=torch.tensor(kv_lens, dtype=torch.int32, device="cpu"),
num_qo_heads=num_query_heads,
num_kv_heads=num_kv_heads,
head_dim=head_size,
page_size=block_size,
q_data_type=dtype,
kv_data_type=dtype,
)
# First call – warmup path (routes through self.plan)
fast_plan_decode(cg_wrapper, **plan_kwargs)
warmup_output = cg_wrapper.run(query, key_value_cache)
torch.testing.assert_close(warmup_output, ref_output, atol=1e-2, rtol=1e-2)
# Second call – fast path (routes through fast_decode_plan from FlashInfer)
fast_plan_decode(cg_wrapper, **plan_kwargs)
fast_output = cg_wrapper.run(query, key_value_cache)
torch.testing.assert_close(fast_output, ref_output, atol=1e-2, rtol=1e-2)
@pytest.mark.parametrize("kv_lens", [[1328, 18, 463], [1, 54, 293, 70]])
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("head_size", HEAD_SIZES)
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("soft_cap", SOFT_CAPS)
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOWS)
@torch.inference_mode
def test_flashinfer_decode_with_paged_kv(
kv_lens: list[int],
num_heads: tuple[int, int],
head_size: int,
dtype: torch.dtype,
block_size: int,
soft_cap: float | None,
sliding_window: int | None,
) -> None:
torch.set_default_device("cuda")
set_random_seed(0)
num_seqs = len(kv_lens)
num_query_heads = num_heads[0]
num_kv_heads = num_heads[1]
assert num_query_heads % num_kv_heads == 0
max_kv_len = max(kv_lens)
scale = head_size**-0.5
query = torch.randn(num_seqs, num_query_heads, head_size, dtype=dtype)
key_value_cache = torch.randn(
NUM_BLOCKS, 2, block_size, num_kv_heads, head_size, dtype=dtype
)
key_cache = key_value_cache[:, 0, :, :, :].squeeze(1)
value_cache = key_value_cache[:, 1, :, :, :].squeeze(1)
max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
block_tables = torch.randint(
0, NUM_BLOCKS, (num_seqs, max_num_blocks_per_seq), dtype=torch.int32
)
kv_indptr = [0]
kv_indices = []
kv_last_page_lens = []
for i in range(num_seqs):
seq_len = kv_lens[i]
assert seq_len > 0
num_blocks = (seq_len + block_size - 1) // block_size
kv_indices.extend(block_tables[i, :num_blocks])
kv_indptr.append(kv_indptr[-1] + num_blocks)
kv_last_page_len = seq_len % block_size
if kv_last_page_len == 0:
kv_last_page_len = block_size
kv_last_page_lens.append(kv_last_page_len)
kv_indptr = torch.tensor(kv_indptr, dtype=torch.int32)
kv_indices = torch.tensor(kv_indices, dtype=torch.int32)
kv_last_page_lens = torch.tensor(kv_last_page_lens, dtype=torch.int32)
workspace_buffer = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
wrapper = flashinfer.BatchDecodeWithPagedKVCacheWrapper(
workspace_buffer, "NHD", use_tensor_cores=True
)
wrapper.plan(
kv_indptr,
kv_indices,
kv_last_page_lens,
num_query_heads,
num_kv_heads,
head_size,
block_size,
"NONE",
window_left=sliding_window - 1 if sliding_window is not None else -1,
q_data_type=dtype,
kv_data_type=dtype,
logits_soft_cap=soft_cap,
)
output = wrapper.run(query, key_value_cache)
ref_output = ref_paged_attn(
query=query,
key_cache=key_cache,
value_cache=value_cache,
query_lens=[1] * num_seqs,
kv_lens=kv_lens,
block_tables=block_tables,
scale=scale,
soft_cap=soft_cap,
sliding_window=sliding_window,
)
(
torch.testing.assert_close(output, ref_output, atol=1e-2, rtol=1e-2),
f"{torch.max(torch.abs(output - ref_output))}",
)
@pytest.mark.parametrize("seq_lens", [[(1, 1328), (5, 18), (129, 463)]])
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("head_size", HEAD_SIZES)
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("soft_cap", SOFT_CAPS)
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOWS)
@torch.inference_mode
def test_flashinfer_prefill_with_paged_kv(
seq_lens: list[tuple[int, int]],
num_heads: tuple[int, int],
head_size: int,
dtype: torch.dtype,
block_size: int,
soft_cap: float | None,
sliding_window: int | None,
) -> None:
torch.set_default_device("cuda")
set_random_seed(0)
num_seqs = len(seq_lens)
query_lens = [x[0] for x in seq_lens]
kv_lens = [x[1] for x in seq_lens]
num_query_heads = num_heads[0]
num_kv_heads = num_heads[1]
assert num_query_heads % num_kv_heads == 0
max_kv_len = max(kv_lens)
scale = head_size**-0.5
query = torch.randn(sum(query_lens), num_query_heads, head_size, dtype=dtype)
key_value_cache = torch.randn(
NUM_BLOCKS, 2, block_size, num_kv_heads, head_size, dtype=dtype
)
key_cache = key_value_cache[:, 0, :, :, :].squeeze(1)
value_cache = key_value_cache[:, 1, :, :, :].squeeze(1)
# Normalize the scale of the key and value caches to mitigate
# numerical instability.
key_cache /= head_size**0.5
value_cache /= head_size**0.5
max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
block_tables = torch.randint(
0, NUM_BLOCKS, (num_seqs, max_num_blocks_per_seq), dtype=torch.int32
)
qo_indptr = [0]
kv_indptr = [0]
kv_indices = []
kv_last_page_lens = []
for i in range(num_seqs):
seq_len = kv_lens[i]
assert seq_len > 0
num_blocks = (seq_len + block_size - 1) // block_size
kv_indices.extend(block_tables[i, :num_blocks])
kv_indptr.append(kv_indptr[-1] + num_blocks)
kv_last_page_len = seq_len % block_size
if kv_last_page_len == 0:
kv_last_page_len = block_size
kv_last_page_lens.append(kv_last_page_len)
qo_indptr.append(qo_indptr[-1] + query_lens[i])
qo_indptr = torch.tensor(qo_indptr, dtype=torch.int32)
kv_indptr = torch.tensor(kv_indptr, dtype=torch.int32)
kv_indices = torch.tensor(kv_indices, dtype=torch.int32)
kv_last_page_lens = torch.tensor(kv_last_page_lens, dtype=torch.int32)
workspace_buffer = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
wrapper = flashinfer.BatchPrefillWithPagedKVCacheWrapper(workspace_buffer, "NHD")
wrapper.plan(
qo_indptr,
kv_indptr,
kv_indices,
kv_last_page_lens,
num_query_heads,
num_kv_heads,
head_size,
block_size,
window_left=sliding_window - 1 if sliding_window is not None else -1,
q_data_type=dtype,
kv_data_type=dtype,
logits_soft_cap=soft_cap,
)
output = wrapper.run(
query,
key_value_cache,
)
ref_output = ref_paged_attn(
query=query,
key_cache=key_cache,
value_cache=value_cache,
query_lens=query_lens,
kv_lens=kv_lens,
block_tables=block_tables,
scale=scale,
soft_cap=soft_cap,
sliding_window=sliding_window,
)
(
torch.testing.assert_close(output, ref_output, atol=5e-2, rtol=1e-2),
f"{torch.max(torch.abs(output - ref_output))}",
)
@pytest.mark.parametrize("seq_lens", [[(1, 132), (5, 18)]])
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("head_size", HEAD_SIZES)
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("soft_cap", SOFT_CAPS)
def test_flashinfer_prefill_with_paged_fp8_kv(
seq_lens: list[tuple[int, int]],
num_heads: tuple[int, int],
head_size: int,
dtype: torch.dtype,
block_size: int,
soft_cap: float | None,
) -> None:
pytest.skip("TODO: fix the accuracy issue")
torch.set_default_device("cuda")
set_random_seed(0)
num_seqs = len(seq_lens)
query_lens = [x[0] for x in seq_lens]
kv_lens = [x[1] for x in seq_lens]
num_query_heads = num_heads[0]
num_kv_heads = num_heads[1]
assert num_query_heads % num_kv_heads == 0
max_kv_len = max(kv_lens)
scale = head_size**-0.5
kv_cache_dtype = torch.float8_e4m3fn
query = torch.randn(sum(query_lens), num_query_heads, head_size, dtype=dtype)
NUM_BLOCKS_FP8 = 2048
key_value_cache = torch.randn(
NUM_BLOCKS_FP8, 2, block_size, num_kv_heads, head_size, dtype=dtype
)
key_cache, value_cache = torch.chunk(key_value_cache, 2, dim=1)
key_cache /= head_size**0.5
value_cache /= head_size**0.5
k_scale = key_cache.amax().item() / 448.0
v_scale = value_cache.amax().item() / 448.0
kv_cache_fp8 = torch.cat([key_cache / k_scale, value_cache / v_scale], dim=1).to(
kv_cache_dtype
)
assert kv_cache_fp8.shape == key_value_cache.shape
max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
block_tables = torch.randint(
0, NUM_BLOCKS_FP8, (num_seqs, max_num_blocks_per_seq), dtype=torch.int32
)
qo_indptr = [0]
kv_indptr = [0]
kv_indices = []
kv_last_page_lens = []
for i in range(num_seqs):
seq_len = kv_lens[i]
assert seq_len > 0
num_blocks = (seq_len + block_size - 1) // block_size
kv_indices.extend(block_tables[i, :num_blocks])
kv_indptr.append(kv_indptr[-1] + num_blocks)
kv_last_page_len = seq_len % block_size
if kv_last_page_len == 0:
kv_last_page_len = block_size
kv_last_page_lens.append(kv_last_page_len)
qo_indptr.append(qo_indptr[-1] + query_lens[i])
qo_indptr = torch.tensor(qo_indptr, dtype=torch.int32)
kv_indptr = torch.tensor(kv_indptr, dtype=torch.int32)
kv_indices = torch.tensor(kv_indices, dtype=torch.int32)
kv_last_page_lens = torch.tensor(kv_last_page_lens, dtype=torch.int32)
workspace_buffer = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
wrapper = flashinfer.BatchPrefillWithPagedKVCacheWrapper(workspace_buffer, "NHD")
wrapper.plan(
qo_indptr,
kv_indptr,
kv_indices,
kv_last_page_lens,
num_query_heads,
num_kv_heads,
head_size,
block_size,
q_data_type=dtype,
kv_data_type=kv_cache_dtype,
logits_soft_cap=soft_cap,
)
output = wrapper.run(query, kv_cache_fp8, k_scale=k_scale, v_scale=v_scale)
ref_output = ref_paged_attn(
query=query,
key_cache=key_cache.squeeze(1),
value_cache=value_cache.squeeze(1),
query_lens=query_lens,
kv_lens=kv_lens,
block_tables=block_tables,
scale=scale,
soft_cap=soft_cap,
)
del query
del block_tables
# verify prefill fp8
(
torch.testing.assert_close(output, ref_output, atol=5e-2, rtol=1e-2),
f"{torch.max(torch.abs(output - ref_output))}",
)
@pytest.mark.parametrize("kv_lens", [[1328, 18, 463], [1, 54, 293, 70]])
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("head_size", HEAD_SIZES)
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("soft_cap", SOFT_CAPS)
@pytest.mark.skip(reason="TODO: fix the accuracy issue")
@torch.inference_mode
def test_flashinfer_decode_with_paged_fp8_kv(
kv_lens: list[int],
num_heads: tuple[int, int],
head_size: int,
dtype: torch.dtype,
block_size: int,
soft_cap: float | None,
) -> None:
# test doesn't work for num_heads = (16,16)
torch.set_default_device("cuda")
set_random_seed(0)
num_seqs = len(kv_lens)
num_query_heads = num_heads[0]
num_kv_heads = num_heads[1]
assert num_query_heads % num_kv_heads == 0
max_kv_len = max(kv_lens)
scale = head_size**-0.5
use_tensor_cores = True
kv_cache_dtype = torch.float8_e4m3fn
query = torch.randn(num_seqs, num_query_heads, head_size, dtype=dtype)
NUM_BLOCKS_FP8 = 2048
key_value_cache = torch.randn(
NUM_BLOCKS_FP8, 2, block_size, num_kv_heads, head_size, dtype=dtype
)
key_cache, value_cache = torch.chunk(key_value_cache, 2, dim=1)
key_cache /= head_size**0.5
value_cache /= head_size**0.5
k_scale = key_cache.amax().item() / 448.0
v_scale = value_cache.amax().item() / 448.0
key_cache_fp8 = (key_cache / k_scale).to(kv_cache_dtype)
value_cache_fp8 = (value_cache / v_scale).to(kv_cache_dtype)
assert key_cache_fp8.shape[1] == 1 and value_cache_fp8.shape[1] == 1
kv_cache_fp8 = torch.cat([key_cache_fp8, value_cache_fp8], dim=1)
max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
block_tables = torch.randint(
0, NUM_BLOCKS_FP8, (num_seqs, max_num_blocks_per_seq), dtype=torch.int32
)
kv_indptr = [0]
kv_indices = []
kv_last_page_lens = []
for i in range(num_seqs):
seq_len = kv_lens[i]
assert seq_len > 0
num_blocks = (seq_len + block_size - 1) // block_size
kv_indices.extend(block_tables[i, :num_blocks])
kv_indptr.append(kv_indptr[-1] + num_blocks)
kv_last_page_len = seq_len % block_size
if kv_last_page_len == 0:
kv_last_page_len = block_size
kv_last_page_lens.append(kv_last_page_len)
kv_indptr = torch.tensor(kv_indptr, dtype=torch.int32)
kv_indices = torch.tensor(kv_indices, dtype=torch.int32)
kv_last_page_lens = torch.tensor(kv_last_page_lens, dtype=torch.int32)
workspace_buffer = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
wrapper = flashinfer.BatchDecodeWithPagedKVCacheWrapper(
workspace_buffer, "NHD", use_tensor_cores=use_tensor_cores
)
wrapper.plan(
kv_indptr,
kv_indices,
kv_last_page_lens,
num_query_heads,
num_kv_heads,
head_size,
block_size,
"NONE",
q_data_type=dtype,
kv_data_type=kv_cache_dtype,
logits_soft_cap=soft_cap,
)
output = wrapper.run(query, kv_cache_fp8, k_scale=k_scale, v_scale=v_scale)
key_cache = key_value_cache[:, 0, :, :, :].squeeze(1)
value_cache = key_value_cache[:, 1, :, :, :].squeeze(1)
ref_output = ref_paged_attn(
query=query,
key_cache=key_cache,
value_cache=value_cache,
query_lens=[1] * num_seqs,
kv_lens=kv_lens,
block_tables=block_tables,
scale=scale,
soft_cap=soft_cap,
)
# Temporary fix: Increasing the tolerance. Seems like a flashinfer issue
(
torch.testing.assert_close(output, ref_output, atol=2e-2, rtol=1e-2),
f"{torch.max(torch.abs(output - ref_output))}",
)
@pytest.mark.skipif(
not any(current_platform.is_device_capability_family(f) for f in (80, 90, 120)),
reason="NVFP4 fa2 path",
)
@pytest.mark.parametrize("layout", ["LBNHC", "LBHNC", "BLNHC", "BLHNC"])
def test_flashinfer_nvfp4_kv_cache_update(default_vllm_config, layout):
from vllm.v1.attention.backends import flashinfer as fi
torch.set_default_device("cuda")
order = KVCacheLayout[layout].stride_order
# [L, B, 2H, N, 64 + 8] NVFP4 pages for two layers; layer 1 is written.
shape = (2, 6, 4, 16, 64 + 8)
layers = torch.zeros([shape[i] for i in order], dtype=torch.uint8)
layers = layers.permute(*(order.index(i) for i in range(5)))
one = torch.ones(())
layer = torch.nn.Module()
layer._k_scale = layer._v_scale = one
kv = torch.randn(2, 96, 2, 128, dtype=torch.bfloat16)
impl = fi.FlashInferImpl(4, 128, 1.0, 2, None, None, "nvfp4")
impl.do_kv_cache_update(layer, kv[0], kv[1], layers[1], torch.arange(96))
data, scales, kv_layout = fi._nvfp4_kv_views(layers[1], 2)
pages = torch.arange(6, dtype=torch.int32)[None]
seq_lens = torch.tensor([96], dtype=torch.int32)
deq = torch.empty_like(kv)[:, None]
flashinfer.nvfp4_kv_dequantize_paged(
data, scales, pages, seq_lens, one, one, *deq, kv_layout
)
assert ((deq[:, 0] - kv).norm() / kv.norm()) < 0.2
assert not layers[0].any()
@@ -1,389 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import torch.nn.functional as F
from torch import Tensor
from vllm.platforms import current_platform
FLASHINFER_WORKSPACE_BUFFER_SIZE = 128 * 1024 * 1024
if not current_platform.is_cuda() or not current_platform.has_device_capability(90):
pytest.skip(
reason="FlashInfer MLA requires CUDA compute capability 9.0 or above.",
allow_module_level=True,
)
else:
from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla
from flashinfer.mla import BatchMLAPagedAttentionWrapper
requires_sm90 = pytest.mark.skipif(
not current_platform.is_device_capability_family(90),
reason="This test requires an SM90 GPU.",
)
requires_sm10x = pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="This test requires an SM10x GPU.",
)
# Deepseek R1 MLA config.
NUM_HEADS = 128
KV_LORA_RANK = 512
QK_NOPE_HEAD_DIM = 128
QK_ROPE_HEAD_DIM = 64
QK_HEAD_DIM = KV_LORA_RANK + QK_ROPE_HEAD_DIM
SCALE = (QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM) ** -0.5
def _make_decode_inputs(bs: int, block_size: int, dtype: torch.dtype):
"""Build valid trtllm MLA decode inputs on the current CUDA device."""
max_seq_len_cap = 1024
seq_lens = [torch.randint(2, max_seq_len_cap, (1,)).item() for _ in range(bs)]
seq_lens[-1] = max_seq_len_cap
max_seq_len = max(seq_lens)
seq_lens_tensor = torch.tensor(seq_lens, dtype=torch.int32)
# Generate block tables with random but unique block IDs
# From https://github.com/flashinfer-ai/flashinfer/pull/1222
blocks_per_seq = (seq_lens_tensor + block_size - 1) // block_size
max_num_blocks_per_seq = max(blocks_per_seq.max().item(), 4)
total_blocks_needed = int(sum(blocks_per_seq))
all_block_ids = torch.randperm(total_blocks_needed)
block_tables = torch.zeros((bs, max_num_blocks_per_seq), dtype=torch.int32)
block_id = 0
for i in range(bs):
num_blocks_needed = blocks_per_seq[i]
block_tables[i, :num_blocks_needed] = all_block_ids[
block_id : block_id + num_blocks_needed
]
block_id += num_blocks_needed
kv_cache = torch.randn(block_tables.numel(), block_size, QK_HEAD_DIM).to(dtype)
q = torch.randn(bs, NUM_HEADS, QK_HEAD_DIM).to(dtype)
return q, kv_cache, block_tables, seq_lens_tensor, max_seq_len
def ref_mla(
out: Tensor, # (bs, num_heads, v_head_dim)
query: Tensor, # (bs, num_heads, head_dim)
kv_cache: Tensor, # (num_blocks, block_size, head_dim)
scale: float,
block_tables: Tensor, # (bs, max_num_blocks)
seq_lens: Tensor, # (bs,)
):
bs, num_heads, v_head_dim = out.shape
head_dim = query.shape[2]
for i in range(bs):
# gather and flatten KV-cache
kv = kv_cache[block_tables[i]] # (max_num_blocks, block_size, head_dim)
kv = kv.view(1, -1, head_dim)[:, : seq_lens[i]] # (1, seq_len, head_dim)
v = kv[:, :, :v_head_dim]
q = query[i].view(num_heads, 1, head_dim)
o = F.scaled_dot_product_attention(q, kv, v, scale=scale, enable_gqa=True)
out[i] = o.view(num_heads, v_head_dim)
return out
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("bs", [1, 2, 4, 16])
@pytest.mark.parametrize("block_size", [32, 64])
@requires_sm10x
def test_flashinfer_mla_decode(dtype: torch.dtype, bs: int, block_size: int):
torch.set_default_device("cuda")
torch.manual_seed(42)
q, kv_cache, block_tables, seq_lens_tensor, max_seq_len = _make_decode_inputs(
bs, block_size, dtype
)
out_ref = q.new_zeros(bs, NUM_HEADS, KV_LORA_RANK)
ref_mla(out_ref, q, kv_cache, SCALE, block_tables, seq_lens_tensor)
workspace_buffer = torch.zeros(
FLASHINFER_WORKSPACE_BUFFER_SIZE,
dtype=torch.uint8,
device=q.device,
)
# Flashinfer MLA expects the query to be of shape
# (bs, q_len_per_request, num_heads, qk_head_dim),
# where q_len_per_request is the MTP query length (=1 without MTP)
q = q.unsqueeze(1)
out_ans = trtllm_batch_decode_with_kv_cache_mla(
query=q,
kv_cache=kv_cache.unsqueeze(1),
workspace_buffer=workspace_buffer,
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
kv_lora_rank=KV_LORA_RANK,
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
block_tables=block_tables,
seq_lens=seq_lens_tensor,
max_seq_len=max_seq_len,
bmm1_scale=SCALE,
)
out_ans = out_ans.squeeze(1)
torch.testing.assert_close(out_ans, out_ref, atol=1e-2, rtol=1e-2)
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn])
@pytest.mark.parametrize("mode", ["tp", "dcp", "hisparse"])
@requires_sm10x
def test_flashinfer_trtllm_sparse_mla_decode_without_rope(dtype, mode, monkeypatch):
"""TP skips preparation; empty DCP contributions and HiSparse outputs vanish."""
import vllm.v1.attention.backends.mla.flashinfer_mla_sparse as backend
from vllm.v1.attention.backends.mla.index_group import HiSparseMLAIndexGroup
torch.set_default_device("cuda")
torch.manual_seed(42)
batch_size, num_heads, block_size, num_blocks, sparse_topk = 3, 8, 64, 4, 2176
valid_lens = [1 if mode == "tp" else 0, 17, 73]
counts = torch.tensor(valid_lens, dtype=torch.int32)
query = torch.randn(batch_size, 1, num_heads, KV_LORA_RANK).to(dtype)
kv_cache = torch.randn(num_blocks, block_size, KV_LORA_RANK).to(dtype)
num_slots = num_blocks * block_size
indices = torch.full((batch_size, sparse_topk), -1, dtype=torch.int32)
for token, length in enumerate(valid_lens):
indices[token, :length] = torch.randperm(num_slots)[:length].int()
empty = counts == 0
reference_indices = indices.clone()
reference_indices[:, 0].masked_fill_(empty, 0)
workspace = torch.empty(FLASHINFER_WORKSPACE_BUFFER_SIZE, dtype=torch.int8)
impl = object.__new__(backend.FlashInferMLASparseImpl)
impl.qk_nope_head_dim = QK_NOPE_HEAD_DIM
impl.kv_lora_rank = KV_LORA_RANK
impl.qk_rope_head_dim = 0
impl.index_group = (
object.__new__(HiSparseMLAIndexGroup) if mode == "hisparse" else None
)
impl._workspace_buffer = workspace
impl.bmm1_scale = QK_NOPE_HEAD_DIM**-0.5
impl.bmm2_scale = 1.0
impl.is_nope_mla = True
impl.need_to_return_lse_for_decode = mode == "dcp"
if mode == "tp":
def unexpected_helper(*args, **kwargs):
pytest.fail("TP must not launch empty-query preparation")
monkeypatch.setattr(
backend, "prepare_sparse_mla_safe_lengths", unexpected_helper
)
out, lse = impl._forward_mqa_kernel(
query.squeeze(1),
kv_cache,
indices,
counts,
layer=None,
block_size=block_size,
is_decode=True,
)
reference = trtllm_batch_decode_with_kv_cache_mla(
query=query,
kv_cache=kv_cache.unsqueeze(1),
workspace_buffer=workspace,
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
kv_lora_rank=KV_LORA_RANK,
qk_rope_head_dim=0,
block_tables=reference_indices.unsqueeze(1),
seq_lens=counts,
max_seq_len=sparse_topk,
sparse_mla_top_k=sparse_topk,
sparse_mla_top_k_lens=counts.clamp(min=1),
bmm1_scale=QK_NOPE_HEAD_DIM**-0.5,
bmm2_scale=1.0,
return_lse=mode == "dcp",
)
if mode == "dcp":
reference, reference_lse = reference
reference_lse = reference_lse.reshape(batch_size, num_heads)
reference_lse.masked_fill_(empty[:, None], float("-inf"))
torch.testing.assert_close(lse, reference_lse, atol=0, rtol=0)
from vllm.v1.attention.ops.dcp import CPTritonContext, correct_attn_out
# The peer owns only our empty selections. Check our contribution after
# the production merge, without zeroing the local attention output.
peer_lse = torch.full_like(lse, float("-inf")).masked_fill(empty[:, None], 0)
out, _ = correct_attn_out(
out, torch.stack((lse, peer_lse)), cp_rank=0, ctx=CPTritonContext()
)
else:
assert lse is None
reference = reference.squeeze(1).masked_fill(empty[:, None, None], 0)
torch.testing.assert_close(out, reference, atol=0, rtol=0)
if dtype == torch.bfloat16:
flat_cache = kv_cache.view(num_slots, KV_LORA_RANK).float()
refs = []
for token, length in enumerate(valid_lens):
selected_kv = flat_cache[reference_indices[token, :length].long()]
scores = torch.einsum("hd,kd->hk", query[token, 0].float(), selected_kv)
probs = torch.softmax(scores * QK_NOPE_HEAD_DIM**-0.5, dim=-1)
refs.append(torch.einsum("hk,kd->hd", probs, selected_kv))
ref = torch.stack(refs).to(torch.bfloat16)
torch.testing.assert_close(out, ref, atol=2e-2, rtol=2e-2)
@requires_sm10x
def test_sparse_mla_safe_lengths():
from vllm.v1.attention.backends.mla.sparse_utils import (
prepare_sparse_mla_safe_lengths,
)
device = "cuda"
num_tokens = 259
# Include a partial tile and expose accidental writes into adjacent storage.
counts_storage = torch.full((num_tokens, 2), 17, dtype=torch.int32, device=device)
counts = counts_storage[:, 0]
counts[::3] = 0
original_counts = counts_storage.clone()
empty = counts == 0
indices_storage = torch.full(
(num_tokens, 256), 42, dtype=torch.int32, device=device
)
indices = indices_storage[:, ::2]
expected_indices = indices_storage.clone()
expected_indices[:, 0].masked_fill_(empty, 0)
safe = prepare_sparse_mla_safe_lengths(indices, counts)
torch.testing.assert_close(safe, counts.clamp(min=1), rtol=0, atol=0)
torch.testing.assert_close(indices_storage, expected_indices, rtol=0, atol=0)
torch.testing.assert_close(counts_storage, original_counts, rtol=0, atol=0)
@requires_sm90
def test_flashinfer_sm90_fp8_mla_decode_without_rope():
"""Hopper FA3 supports BF16 queries over an FP8 cache without KPE."""
torch.manual_seed(42)
device = torch.device("cuda")
batch_size = 2
num_heads = 16
page_size = 16
num_pages = 6
q_nope = torch.randn(
batch_size,
num_heads,
KV_LORA_RANK,
dtype=torch.bfloat16,
device=device,
)
q_pe = torch.empty(
batch_size,
num_heads,
0,
dtype=torch.bfloat16,
device=device,
)
ckv = torch.randn(
num_pages,
page_size,
KV_LORA_RANK,
device=device,
)
fp8_max = torch.finfo(torch.float8_e4m3fn).max
ckv_scale = ckv.abs().max().item() / fp8_max
ckv_fp8 = (ckv / ckv_scale).clamp(-fp8_max, fp8_max).to(torch.float8_e4m3fn)
scale_bf16 = torch.tensor(ckv_scale, dtype=torch.bfloat16, device=device)
ckv_ref = ckv_fp8.to(torch.bfloat16) * scale_bf16
kpe_fp8 = torch.empty(
num_pages,
page_size,
0,
dtype=torch.float8_e4m3fn,
device=device,
)
kpe_ref = torch.empty(
num_pages,
page_size,
0,
dtype=torch.bfloat16,
device=device,
)
qo_indptr = torch.tensor([0, 1, 2], dtype=torch.int32, device=device)
kv_indptr = torch.tensor([0, 3, 5], dtype=torch.int32, device=device)
kv_indices = torch.tensor([4, 1, 3, 0, 5], dtype=torch.int32, device=device)
kv_lens = torch.tensor([45, 29], dtype=torch.int32, device=device)
sm_scale = QK_NOPE_HEAD_DIM**-0.5
def run(
ckv_cache: torch.Tensor,
kpe_cache: torch.Tensor,
**kwargs,
) -> torch.Tensor:
workspace = torch.empty(
FLASHINFER_WORKSPACE_BUFFER_SIZE,
dtype=torch.uint8,
device=device,
)
wrapper = BatchMLAPagedAttentionWrapper(workspace, backend="fa3")
wrapper.plan(
qo_indptr,
kv_indptr,
kv_indices,
kv_lens,
num_heads,
KV_LORA_RANK,
0,
page_size,
False,
sm_scale,
q_data_type=torch.bfloat16,
kv_data_type=ckv_cache.dtype,
)
return wrapper.run(q_nope, q_pe, ckv_cache, kpe_cache, **kwargs)
out_ref = run(ckv_ref, kpe_ref)
out = run(ckv_fp8, kpe_fp8, ckv_scale=ckv_scale, kpe_scale=1.0)
torch.testing.assert_close(out, out_ref, atol=2e-2, rtol=2e-2)
@requires_sm10x
def test_flashinfer_mla_decode_workspace_supports_autotune():
"""VLLM's FlashInfer MLA decode workspace must be int8 for autotuning.
Model Runner V2's warmup autotunes ``trtllm_batch_decode_mla``, which makes
the FlashInfer autotuner enumerate the CuteDSL tactic. That tactic asserts
``workspace_buffer.dtype == torch.int8``; the trtllm-gen path (used for
normal, non-autotuned inference) instead views the buffer as uint8, so a
uint8 workspace only fails once the autotuner tries CuteDSL. That regressed
every DeepSeek MLA test on Blackwell under V2 with
``workspace_buffer must be torch.int8`` (vllm-project/vllm#46646).
"""
from flashinfer.autotuner import autotune
from vllm.v1.attention.backends.mla.flashinfer_mla import _get_workspace_buffer
torch.set_default_device("cuda")
torch.manual_seed(0)
workspace_buffer = _get_workspace_buffer(return_lse=False)
assert workspace_buffer.dtype == torch.int8
q, kv_cache, block_tables, seq_lens_tensor, max_seq_len = _make_decode_inputs(
bs=1, block_size=64, dtype=torch.bfloat16
)
# Under the autotuner the CuteDSL tactic is instantiated with our workspace;
# a uint8 buffer raises AssertionError here, an int8 buffer succeeds.
with torch.inference_mode(), autotune(True):
trtllm_batch_decode_with_kv_cache_mla(
query=q.unsqueeze(1),
kv_cache=kv_cache.unsqueeze(1),
workspace_buffer=workspace_buffer,
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
kv_lora_rank=KV_LORA_RANK,
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
block_tables=block_tables,
seq_lens=seq_lens_tensor,
max_seq_len=max_seq_len,
bmm1_scale=SCALE,
)
@@ -1,842 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
@pytest.mark.parametrize("sm120", [False, True])
def test_deepseek_v4_c128a_adaptive_width_has_capture_stable_stride(
monkeypatch: pytest.MonkeyPatch,
sm120: bool,
):
from vllm.models.deepseek_v4 import sparse_mla
from vllm.platforms.interface import DeviceCapability
monkeypatch.setattr(
sparse_mla.current_platform,
"get_device_capability",
lambda: DeviceCapability(12, 0) if sm120 else DeviceCapability(10, 0),
)
device = torch.device("cuda")
capacity_width = 512
global_decode_buffer = torch.empty(
(2, capacity_width), dtype=torch.int32, device=device
)
prefill_buffer = torch.empty_like(global_decode_buffer)
kwargs = dict(
positions=torch.tensor([255, 511, 383, 639], device=device),
compress_ratio=128,
num_decode_tokens=2,
token_to_req_indices=torch.tensor(
[0, 1, 0, 1], dtype=torch.int32, device=device
),
block_table=torch.tensor([[3], [5]], dtype=torch.int32, device=device),
block_size=capacity_width,
slot_mapping=torch.arange(4, dtype=torch.int64, device=device),
global_decode_buffer=global_decode_buffer,
decode_lens_buffer=torch.empty(2, dtype=torch.int32, device=device),
prefill_buffer=prefill_buffer,
)
captured_decode, _, captured_prefill = sparse_mla.build_c128a_topk_metadata(
max_compressed_tokens=256,
**kwargs,
)
# SM120 keeps the decode view contiguous across the full buffer width;
# other backends get the active-width slice. The prefill view is always
# narrowed.
expected_decode_width = capacity_width if sm120 else 256
assert captured_decode.shape == (2, expected_decode_width)
assert captured_prefill.shape == (2, 256)
assert captured_decode.stride(0) == captured_prefill.stride(0) == capacity_width
assert captured_decode.is_contiguous() == sm120
captured_rows = torch.empty((4, 4), dtype=torch.int32, device=device)
captured_rows[:2].copy_(captured_decode[:, :4])
captured_rows[2:].copy_(captured_prefill[:, :4])
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
captured_rows[:2].copy_(captured_decode[:, :4])
captured_rows[2:].copy_(captured_prefill[:, :4])
global_decode_buffer.fill_(-99)
prefill_buffer.fill_(-99)
sparse_mla.build_c128a_topk_metadata(
max_compressed_tokens=128,
**kwargs,
)
graph.replay()
assert captured_rows.cpu().tolist() == [
[1536, 1537, -1, -1],
[2560, 2561, 2562, 2563],
[0, 1, 2, -1],
[0, 1, 2, 3],
]
assert torch.all(global_decode_buffer[:, 128:] == -99)
assert torch.all(prefill_buffer[:, 128:] == -99)
def test_sparse_flashmla_metadata_smoke():
import vllm.v1.attention.ops.flashmla as fm
ok, reason = fm.is_flashmla_sparse_supported()
if not ok:
pytest.skip(reason)
device = torch.device("cuda")
batch_size = 1
seqlen_q = 1
num_heads_q = 128
num_heads_k = 1
q_seq_per_hk = seqlen_q * num_heads_q // num_heads_k
topk = 128
cache_seqlens = torch.zeros(batch_size, dtype=torch.int32, device=device)
tile_md, num_splits = fm.get_mla_metadata(
cache_seqlens,
q_seq_per_hk,
num_heads_k,
num_heads_q=num_heads_q,
topk=topk,
is_fp8_kvcache=True,
)
assert isinstance(tile_md, fm.FlashMLASchedMeta)
assert tile_md.tile_scheduler_metadata is None
assert tile_md.num_splits is None
assert num_splits is None
def test_sparse_flashmla_decode_matches_cache_writer_scales():
import vllm.v1.attention.ops.flashmla as fm
from vllm import _custom_ops as ops
ok, reason = fm.is_flashmla_sparse_supported()
if not ok:
pytest.skip(reason)
device = torch.device("cuda")
batch_size = 1
seqlen_q = 1
num_heads_q = 64
head_dim_k = 576
head_dim_v = 512
num_heads_k = 1
page_block_size = 64
bytes_per_token = 656
topk = 128
# Metadata
q_seq_per_hk = seqlen_q * num_heads_q // num_heads_k
# q_heads_per_hk = num_heads_q // num_heads_k
cache_seqlens = torch.ones(batch_size, dtype=torch.int32, device=device)
tile_md, num_splits = fm.get_mla_metadata(
cache_seqlens,
q_seq_per_hk,
num_heads_k,
num_heads_q=num_heads_q,
topk=topk,
is_fp8_kvcache=True,
)
# Inputs
q = torch.zeros(
(batch_size, seqlen_q, num_heads_q, head_dim_k),
dtype=torch.bfloat16,
device=device,
)
cache_rows = torch.zeros(
(1, page_block_size, bytes_per_token), dtype=torch.uint8, device=device
)
# 336 / 448 = 0.75 distinguishes arbitrary FP32 from power-of-two scales.
kv_c = torch.full((1, 512), 336.0, dtype=torch.bfloat16, device=device)
kv_c[:, -128:] = 0
ops.concat_and_cache_mla(
kv_c,
torch.zeros((1, 64), dtype=torch.bfloat16, device=device),
cache_rows,
torch.zeros(1, dtype=torch.int64, device=device),
"fp8_ds_mla",
torch.ones(1, dtype=torch.float32, device=device),
)
scales = cache_rows[0, 0].view(torch.float32)[128:132]
expected_scales = torch.tensor([1.0, 1.0, 1.0, 2**-13], device=device)
torch.testing.assert_close(scales, expected_scales, rtol=0, atol=0)
cached_nope = cache_rows[0, 0, :512].view(torch.float8_e4m3fn).float()
expected = (cached_nope * scales.repeat_interleave(128)).to(torch.bfloat16)
k_cache = cache_rows.unsqueeze(2)
indices = torch.full(
(batch_size, seqlen_q, topk), -1, dtype=torch.int32, device=device
)
indices[..., 0] = 0
# SM90 sparse decode only supports a fixed top-k width. Padding with -1
# exercises the same cache data on SM90 and SM100 without topk_length.
block_table = torch.zeros((batch_size, 128), dtype=torch.int32, device=device)
out, lse = fm.flash_mla_with_kvcache(
q,
k_cache,
block_table,
cache_seqlens,
head_dim_v,
tile_md,
num_splits,
indices=indices,
is_fp8_kvcache=True,
)
assert out.shape[0] == batch_size
assert out.shape[-1] == head_dim_v
assert lse.shape[0] == batch_size
torch.testing.assert_close(
out, expected.view(1, 1, 1, 512).expand_as(out), rtol=0, atol=0
)
@pytest.mark.parametrize("h_q", [64, 128])
def test_sparse_flashmla_prefill_smoke(h_q: int):
import vllm.v1.attention.ops.flashmla as fm
ok, reason = fm.is_flashmla_sparse_supported()
if not ok:
pytest.skip(reason)
device = torch.device("cuda")
torch.manual_seed(0)
s_q = 1
s_kv = 8
h_kv = 1
d_qk = 576
d_v = 512
topk = 128
q = torch.randn((s_q, h_q, d_qk), dtype=torch.bfloat16, device=device)
kv = torch.randn((s_kv, h_kv, d_qk), dtype=torch.bfloat16, device=device)
indices = torch.randint(s_kv, (s_q, h_kv, topk), dtype=torch.int32, device=device)
reference_indices = indices.clone()
reference_indices[..., 1:] = -1
kwargs = {"topk_length": torch.ones(1, dtype=torch.int32, device=device)}
reference = fm.flash_mla_sparse_fwd(q, kv, reference_indices, 1.0, d_v, **kwargs)
actual = fm.flash_mla_sparse_fwd(q, kv, indices, 1.0, d_v, **kwargs)
for actual_tensor, reference_tensor in zip(actual, reference):
torch.testing.assert_close(actual_tensor, reference_tensor, rtol=0, atol=0)
assert actual[0].shape == (s_q, h_q, d_v)
def test_deepseek_v4_prefill_chunk_planning_expands_for_short_sequences():
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
metadata = DeepseekSparseSWAMetadata(
block_table=torch.empty(0, dtype=torch.int32),
slot_mapping=torch.empty(0, dtype=torch.int32),
block_size=64,
num_prefills=5,
prefill_seq_lens_cpu=torch.tensor([80, 96, 112, 128, 144], dtype=torch.int32),
prefill_query_lens_cpu=torch.tensor([4, 4, 4, 4, 4], dtype=torch.int32),
prefill_window_size=64,
prefill_max_model_len=1024,
prefill_max_num_batched_tokens=128,
)
chunk_plan = metadata.get_prefill_chunk_plan(compress_ratio=4, prefill_chunk_size=4)
# the adaptive plan keeps all 5 in one chunk
assert chunk_plan == [(0, 5, 36, 103)]
def test_flashinfer_sparse_indices_cache(monkeypatch):
from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as flashinfer_mod
from vllm.models.deepseek_v4.sparse_mla import DeepseekV4FlashMLAMetadata
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
builder_calls = 0
def fake_build(*args, **kwargs):
nonlocal builder_calls
builder_calls += 1
return (
torch.tensor([[builder_calls]], dtype=torch.int32),
torch.tensor([builder_calls], dtype=torch.int32),
)
monkeypatch.setattr(
flashinfer_mod, "build_flashinfer_mixed_sparse_indices", fake_build
)
def make_attn(compress_ratio: int, topk_width: int):
attn = object.__new__(flashinfer_mod.DeepseekV4FlashInferMLAAttention)
attn.compress_ratio = compress_ratio
attn.window_size = 4
attn.topk_indices_buffer = torch.tensor(
[[0, 1], [2, 3], [4, 5]], dtype=torch.int32
)[:, :topk_width]
return attn
def make_swa_metadata():
return DeepseekSparseSWAMetadata(
block_table=torch.tensor([[0, 1], [2, 3]], dtype=torch.int32),
slot_mapping=torch.tensor([0, 1], dtype=torch.int64),
block_size=64,
seq_lens=torch.tensor([8, 10], dtype=torch.int32),
query_start_loc=torch.tensor([0, 1, 3], dtype=torch.int32),
query_start_loc_cpu=torch.tensor([0, 1, 3], dtype=torch.int32),
token_to_req_indices=torch.tensor([0, 1, 1], dtype=torch.int32),
decode_swa_indices=torch.tensor([[5, 6, -1, -1]], dtype=torch.int32),
decode_swa_lens=torch.tensor([2], dtype=torch.int32),
decode_swa_width=4,
is_valid_token=torch.tensor([True], dtype=torch.bool),
num_decodes=1,
num_prefills=1,
num_decode_tokens=1,
num_prefill_tokens=2,
)
def make_flashmla_metadata():
return DeepseekV4FlashMLAMetadata(
num_reqs=2,
max_query_len=2,
max_seq_len=10,
num_actual_tokens=3,
query_start_loc=torch.tensor([0, 1, 3], dtype=torch.int32),
slot_mapping=torch.tensor([0, 1, 2], dtype=torch.int64),
block_table=torch.tensor([[0, 1], [2, 3]], dtype=torch.int32),
req_id_per_token=torch.tensor([0, 1, 1], dtype=torch.int32),
block_size=256,
topk_tokens=2,
c128a_global_decode_topk_indices=torch.tensor(
[[[9, 10]]], dtype=torch.int32
),
c128a_decode_topk_lens=torch.tensor([2], dtype=torch.int32),
c128a_prefill_topk_indices=torch.tensor(
[[0, 1], [1, 2]], dtype=torch.int32
),
)
swa_attn = make_attn(1, 0)
swa_metadata = make_swa_metadata()
_, _, sparse_indices_first, sparse_lens_first = (
swa_attn._build_sparse_index_metadata(
kv_cache=None,
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
swa_metadata=swa_metadata,
attn_metadata=None,
swa_only=True,
)
)
_, _, sparse_indices_second, sparse_lens_second = (
swa_attn._build_sparse_index_metadata(
kv_cache=None,
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
swa_metadata=swa_metadata,
attn_metadata=None,
swa_only=True,
)
)
assert builder_calls == 1
assert sparse_indices_first is sparse_indices_second
assert sparse_lens_first is sparse_lens_second
c128a_attn = make_attn(128, 2)
c128a_metadata = make_swa_metadata()
c128a_flashmla_md = make_flashmla_metadata()
_, _, sparse_indices_first, sparse_lens_first = (
c128a_attn._build_sparse_index_metadata(
kv_cache=torch.empty((1, 2, 512), dtype=torch.bfloat16),
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
swa_metadata=c128a_metadata,
attn_metadata=c128a_flashmla_md,
swa_only=False,
)
)
_, _, sparse_indices_second, sparse_lens_second = (
c128a_attn._build_sparse_index_metadata(
kv_cache=torch.empty((1, 2, 512), dtype=torch.bfloat16),
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
swa_metadata=c128a_metadata,
attn_metadata=c128a_flashmla_md,
swa_only=False,
)
)
assert builder_calls == 2
assert sparse_indices_first is sparse_indices_second
assert sparse_lens_first is sparse_lens_second
c4a_attn = make_attn(4, 2)
c4a_metadata = make_swa_metadata()
c4a_flashmla_md = make_flashmla_metadata()
c4a_flashmla_md.c128a_global_decode_topk_indices = None
c4a_flashmla_md.c128a_decode_topk_lens = None
c4a_flashmla_md.c128a_prefill_topk_indices = None
_, _, sparse_indices_third, sparse_lens_third = (
c4a_attn._build_sparse_index_metadata(
kv_cache=torch.empty((1, 2, 512), dtype=torch.bfloat16),
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
swa_metadata=c4a_metadata,
attn_metadata=c4a_flashmla_md,
swa_only=False,
)
)
_, _, sparse_indices_fourth, sparse_lens_fourth = (
c4a_attn._build_sparse_index_metadata(
kv_cache=torch.empty((1, 2, 512), dtype=torch.bfloat16),
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
swa_metadata=c4a_metadata,
attn_metadata=c4a_flashmla_md,
swa_only=False,
)
)
assert builder_calls == 4
assert sparse_indices_third is not sparse_indices_fourth
assert sparse_lens_third is not sparse_lens_fourth
def test_flashinfer_sparse_index_preserves_logical_window(monkeypatch):
from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as flashinfer_mod
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
captured_shapes_and_windows: list[tuple[int, int]] = []
def fake_build(*args, **kwargs):
# window_size is the 12th positional arg of
# build_flashinfer_mixed_sparse_indices.
captured_shapes_and_windows.append((args[0].shape[-1], args[11]))
num_tokens = args[0].shape[0] + args[3].shape[0]
return (
torch.zeros((num_tokens, 1), dtype=torch.int32),
torch.zeros((num_tokens,), dtype=torch.int32),
)
monkeypatch.setattr(
flashinfer_mod, "build_flashinfer_mixed_sparse_indices", fake_build
)
attn = object.__new__(flashinfer_mod.DeepseekV4FlashInferMLAAttention)
attn.compress_ratio = 1
attn.window_size = 4
attn.topk_indices_buffer = torch.zeros((4, 0), dtype=torch.int32)
wide_width = 8
wide_indices = torch.full((1, wide_width), -1, dtype=torch.int32)
wide_indices[0, :2] = torch.tensor([5, 6], dtype=torch.int32)
wide_metadata = DeepseekSparseSWAMetadata(
block_table=torch.tensor([[0, 1]], dtype=torch.int32),
slot_mapping=torch.tensor([0], dtype=torch.int64),
block_size=64,
seq_lens=torch.tensor([8], dtype=torch.int32),
query_start_loc=torch.tensor([0, 1], dtype=torch.int32),
query_start_loc_cpu=torch.tensor([0, 1], dtype=torch.int32),
token_to_req_indices=torch.tensor([0], dtype=torch.int32),
decode_swa_indices=wide_indices,
decode_swa_lens=torch.tensor([2], dtype=torch.int32),
decode_swa_width=wide_width,
is_valid_token=torch.tensor([True], dtype=torch.bool),
num_decodes=1,
num_prefills=0,
num_decode_tokens=1,
num_prefill_tokens=0,
)
attn._build_sparse_index_metadata(
kv_cache=None,
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
swa_metadata=wide_metadata,
attn_metadata=None,
swa_only=True,
)
assert captured_shapes_and_windows == [(wide_width, attn.window_size)]
empty_width = 8
empty_metadata = DeepseekSparseSWAMetadata(
block_table=torch.tensor([[0, 1]], dtype=torch.int32),
slot_mapping=torch.tensor([0, 1], dtype=torch.int64),
block_size=64,
seq_lens=torch.tensor([8], dtype=torch.int32),
query_start_loc=torch.tensor([0, 2], dtype=torch.int32),
query_start_loc_cpu=torch.tensor([0, 2], dtype=torch.int32),
token_to_req_indices=torch.tensor([0, 0], dtype=torch.int32),
decode_swa_indices=torch.empty((0, 1, empty_width), dtype=torch.int32),
decode_swa_lens=torch.empty((0,), dtype=torch.int32),
decode_swa_width=empty_width,
is_valid_token=torch.tensor([True, True], dtype=torch.bool),
num_decodes=0,
num_prefills=1,
num_decode_tokens=0,
num_prefill_tokens=2,
)
attn._build_sparse_index_metadata(
kv_cache=None,
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
swa_metadata=empty_metadata,
attn_metadata=None,
swa_only=True,
)
assert captured_shapes_and_windows == [
(wide_width, attn.window_size),
(empty_width, attn.window_size),
]
def test_flashinfer_mixed_sparse_indices_separates_window_and_padded_width():
from vllm.models.deepseek_v4.common.ops.cache_utils import (
build_flashinfer_mixed_sparse_indices,
)
device = torch.device("cuda")
padded_width = 8
logical_window = 4
sparse_indices, sparse_lens = build_flashinfer_mixed_sparse_indices(
decode_swa_indices=torch.empty(
(0, padded_width), dtype=torch.int32, device=device
),
decode_compressed_indices=None,
decode_compressed_topk_lens=None,
prefill_topk_indices=torch.empty((1, 0), dtype=torch.int32, device=device),
query_start_loc=torch.tensor([0, 1], dtype=torch.int32, device=device),
seq_lens=torch.tensor([logical_window], dtype=torch.int32, device=device),
token_to_req_indices=torch.tensor([0], dtype=torch.int32, device=device),
swa_block_table=torch.tensor([[0]], dtype=torch.int32, device=device),
swa_block_size=64,
compressed_block_table=None,
compressed_block_size=64,
window_size=logical_window,
compress_ratio=1,
topk=0,
)
assert sparse_indices.shape == (1, padded_width)
assert sparse_indices[0].cpu().tolist() == [0, 1, 2, 3, -1, -1, -1, -1]
assert sparse_lens.cpu().tolist() == [padded_width]
def test_flashinfer_mixed_sparse_indices_noncausal_rows_have_no_active_gaps():
"""A DSpark non-causal row keeps -1 out of the kernel's active ranges.
Each draft token's causal window holds its first visible entries, the rest
of the block starts at column `window`, and the compressed entries follow
directly; `sparse_topk_lens` ends right after them.
"""
from vllm.models.deepseek_v4.common.ops.cache_utils import (
build_flashinfer_mixed_sparse_indices,
)
device = torch.device("cuda")
window, width = 4, 8
# One request: 1 context token, a 2-token draft block, slots 0..2 visible.
decode_swa = torch.full((2, width), -1, dtype=torch.int32, device=device)
decode_swa[:, :3] = torch.arange(3, dtype=torch.int32, device=device)
compressed = torch.tensor(
[[100, 101, -1, -1]] * 2, dtype=torch.int32, device=device
)
sparse_indices, sparse_lens = build_flashinfer_mixed_sparse_indices(
decode_swa_indices=decode_swa,
decode_compressed_indices=compressed,
decode_compressed_topk_lens=torch.tensor(
[2, 2], dtype=torch.int32, device=device
),
prefill_topk_indices=torch.empty((0, 4), dtype=torch.int32, device=device),
query_start_loc=torch.tensor([0, 2], dtype=torch.int32, device=device),
seq_lens=torch.tensor([3], dtype=torch.int32, device=device),
token_to_req_indices=torch.tensor([0, 0], dtype=torch.int32, device=device),
swa_block_table=torch.tensor([[0]], dtype=torch.int32, device=device),
swa_block_size=64,
compressed_block_table=torch.tensor([[0]], dtype=torch.int32, device=device),
compressed_block_size=64,
window_size=window,
compress_ratio=128,
topk=0,
)
assert sparse_indices.cpu().tolist() == [
# pos 1: window [0, 1]; slot 2 spills to column 4; compressed after it.
[0, 1, -1, -1, 2, 100, 101, -1, -1, -1, -1, -1],
# pos 2: window [0, 1, 2]; nothing spills; compressed at column 4.
[0, 1, 2, -1, 100, 101, -1, -1, -1, -1, -1, -1],
]
assert sparse_lens.cpu().tolist() == [7, 6]
def test_flashinfer_mixed_sparse_indices_noncausal_pad_rows_keep_min_length():
"""A CUDA-graph pad token of the DSpark graph (a zero-length request) gets
a causal pad row's shape instead of negative spill arithmetic."""
from vllm.models.deepseek_v4.common.ops.cache_utils import (
build_flashinfer_mixed_sparse_indices,
)
device = torch.device("cuda")
window, width = 128, 192
decode_swa = torch.full((3, width), -1, dtype=torch.int32, device=device)
decode_swa[:2, :3] = torch.arange(3, dtype=torch.int32, device=device)
sparse_indices, sparse_lens = build_flashinfer_mixed_sparse_indices(
decode_swa_indices=decode_swa,
decode_compressed_indices=None,
decode_compressed_topk_lens=None,
prefill_topk_indices=torch.empty((0, 0), dtype=torch.int32, device=device),
# Request 1 is the padding: no query tokens, seq_len 0.
query_start_loc=torch.tensor([0, 2, 2], dtype=torch.int32, device=device),
seq_lens=torch.tensor([3, 0], dtype=torch.int32, device=device),
token_to_req_indices=torch.tensor([0, 0, 1], dtype=torch.int32, device=device),
swa_block_table=torch.zeros((2, 1), dtype=torch.int32, device=device),
swa_block_size=64,
compressed_block_table=None,
compressed_block_size=64,
window_size=window,
compress_ratio=1,
topk=0,
)
assert sparse_lens.cpu().tolist() == [129, 128, 128]
assert (sparse_indices[2] == -1).all()
@pytest.mark.parametrize("model", ["deepseek_v4", "deepseek_v41"])
@pytest.mark.parametrize("page_padding", [0, 64])
def test_flashinfer_sparse_forward_reads_packed_kv_rows(model, page_padding):
"""Decode and prefill must read storage rows after vLLM remaps packed pages."""
import importlib
from vllm.platforms import current_platform
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
if not current_platform.is_device_capability_family(100):
pytest.skip("Requires FlashInfer TRTLLM sparse MLA on SM100/SM103")
mod = importlib.import_module(f"vllm.models.{model}.nvidia.flashinfer_sparse")
device = "cuda"
block_size = 64
storage = torch.full(
(3, block_size + page_padding, 512), 99, dtype=torch.bfloat16, device=device
)
cache = storage[:, :block_size]
cache[1].fill_(2)
cache[2].fill_(3)
attn = object.__new__(mod.DeepseekV4FlashInferMLAAttention)
attn.compress_ratio = 1
attn.window_size = 128
attn.scale = 512**-0.5
attn.kv_cache_torch_dtype = torch.bfloat16
attn.attn_sink = torch.full((64,), -float("inf"), device=device)
attn.topk_indices_buffer = torch.empty((3, 0), dtype=torch.int32, device=device)
decode_indices = torch.full((1, 128), -1, dtype=torch.int32, device=device)
decode_indices[0, :4] = torch.arange(64, 68, dtype=torch.int32, device=device)
query_start = torch.tensor([0, 1, 3], dtype=torch.int32)
metadata = DeepseekSparseSWAMetadata(
block_table=torch.tensor([[1], [2]], dtype=torch.int32, device=device),
slot_mapping=torch.tensor([67, 130, 131], device=device),
block_size=block_size,
seq_lens=torch.tensor([4, 4], dtype=torch.int32, device=device),
query_start_loc=query_start.to(device),
query_start_loc_cpu=query_start,
token_to_req_indices=torch.tensor([0, 1, 1], dtype=torch.int32, device=device),
decode_swa_indices=decode_indices,
decode_swa_lens=torch.tensor([4], dtype=torch.int32, device=device),
decode_swa_width=128,
is_valid_token=torch.ones(3, dtype=torch.bool, device=device),
replay_start=torch.zeros(2, dtype=torch.int32, device=device),
num_decodes=1,
num_prefills=1,
num_decode_tokens=1,
num_prefill_tokens=2,
max_decode_query_len=1,
)
query = torch.zeros((3, 64, 512), dtype=torch.bfloat16, device=device)
output = torch.empty_like(query)
attn._forward(query, None, cache, metadata, None, True, output)
expected = torch.tensor([2, 3, 3], dtype=output.dtype, device=device)
torch.testing.assert_close(output, expected[:, None, None].expand_as(output))
def _make_rope_quant_attn(mod, num_rows: int, device: str):
"""A bare DSv4 FlashInfer layer set up for RopeQuant: 128 heads, fp8 KV."""
from types import SimpleNamespace
heads, groups, rank = 128, 16, 256
angles = torch.arange(2048, device=device, dtype=torch.float32)[:, None] * 0.05
angles = angles + torch.arange(32, device=device)[None] * 0.1
attn = object.__new__(mod.DeepseekV4FlashInferMLAAttention)
attn.compress_ratio = 1
attn.window_size = 128
attn.scale = 512**-0.5
attn.kv_cache_torch_dtype = torch.float8_e4m3fn
attn._flashinfer_fp8_bmm1_scale = attn.scale
attn._flashinfer_fp8_bmm2_scale = 1.0
attn.attn_sink = torch.linspace(-1, 1, heads, device=device)
attn.padded_heads = heads
attn.topk_indices_buffer = torch.empty(
(num_rows, 0), dtype=torch.int32, device=device
)
attn.rotary_emb = SimpleNamespace(
cos_sin_cache=torch.cat((angles.cos(), angles.sin()), -1).contiguous()
)
attn.n_local_heads, attn.n_local_groups, attn.o_lora_rank = heads, groups, rank
attn.nope_head_dim, attn.rope_head_dim = 448, 64
attn._einsum_recipe, attn._tma_aligned_scales = (1, 1, 128), True
attn.wo_a = SimpleNamespace(
weight=(torch.randn((groups, rank, 4096), device=device) * 0.05).to(
torch.float8_e4m3fn
),
weight_scale=torch.ones((groups, rank, 32), device=device),
)
attn.wo_b = lambda z: z
return attn
def _rope_quant_vs_unfused(attn, q, cache, metadata, positions, num_tokens):
"""Run `_forward` unfused and fused; return (bf16 attention, z_unfused, z_fused)."""
from vllm.models.deepseek_v4.nvidia.ops.o_proj import rope_quant_attn_out
heads, groups = attn.n_local_heads, attn.n_local_groups
unfused = torch.empty(q.shape, dtype=torch.bfloat16, device=q.device)
attn._forward(q, None, cache, metadata, None, True, unfused)
fused = rope_quant_attn_out(q.shape[0], q.device)
metadata.flashinfer_sparse_index_cache.clear()
attn._forward(q, None, cache, metadata, None, True, fused)
z_unfused = attn._o_proj(unfused[:num_tokens, :heads], positions)
z_fused = attn._o_proj(fused, positions)[:num_tokens]
return unfused[:num_tokens], z_unfused, z_fused.view(num_tokens, groups, -1)
def _assert_projects_alike(z_unfused, z_fused):
z_unfused = z_unfused.float().view_as(z_fused)
rel = (z_fused.float() - z_unfused).norm() / z_unfused.norm()
assert rel < 2e-2, rel
def _skip_unless_rope_quant():
from vllm.platforms import current_platform
if not current_platform.is_device_capability_family(100):
pytest.skip("Requires FlashInfer TRTLLM sparse MLA on SM100/SM103")
def test_flashinfer_rope_quant_matches_unfused_o_proj():
"""RopeQuant's fused output must project like the bf16 + inv-RoPE/quant path.
One mixed step (two decodes, a three-token prefill) plus CUDA-graph pad rows,
so both calls run over the full buffer with un-rebased cum_seq_lens_q.
"""
from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as mod
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
_skip_unless_rope_quant()
torch.manual_seed(0)
device = "cuda"
block_size = 64
q_lens, seq_lens = [1, 1, 3], [5, 9, 70]
num_tokens, num_rows = sum(q_lens), sum(q_lens) + 3
cache = torch.randn((3, block_size, 512), device=device).to(torch.float8_e4m3fn)
attn = _make_rope_quant_attn(mod, num_rows, device)
decode_indices = torch.full((2, 128), -1, dtype=torch.int32, device=device)
for i, seq_len in enumerate(seq_lens[:2]):
decode_indices[i, :seq_len] = torch.arange(
i * block_size, i * block_size + seq_len, device=device
)
query_start = torch.tensor([0, 1, 2, 5], dtype=torch.int32)
metadata = DeepseekSparseSWAMetadata(
block_table=torch.tensor([[0], [1], [2]], dtype=torch.int32, device=device),
slot_mapping=torch.zeros(num_tokens, dtype=torch.int64, device=device),
block_size=block_size,
seq_lens=torch.tensor(seq_lens, dtype=torch.int32, device=device),
query_start_loc=query_start.to(device),
query_start_loc_cpu=query_start,
token_to_req_indices=torch.tensor(
[0, 1, 2, 2, 2], dtype=torch.int32, device=device
),
decode_swa_indices=decode_indices,
decode_swa_lens=torch.tensor(seq_lens[:2], dtype=torch.int32, device=device),
decode_swa_width=128,
is_valid_token=torch.ones(num_tokens, dtype=torch.bool, device=device),
num_decodes=2,
num_prefills=1,
num_decode_tokens=2,
num_prefill_tokens=3,
max_decode_query_len=1,
)
positions = torch.tensor([4, 8, 67, 68, 69], device=device)
q = torch.randn((num_rows, 128, 512), device=device).to(torch.float8_e4m3fn)
_, z_unfused, z_fused = _rope_quant_vs_unfused(
attn, q, cache, metadata, positions, num_tokens
)
_assert_projects_alike(z_unfused, z_fused)
@pytest.mark.parametrize("context_len", [0, 20, 127, 900])
def test_flashinfer_dspark_noncausal_block_sees_future_tokens(context_len):
"""Every DSpark draft token attends to the whole block, fused or not.
The kernel counts every entry of its active index ranges as a key, so the
builder must place the block around each query's causal window without -1
gaps, down to an empty context.
"""
from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as mod
from vllm.v1.attention.backends.mla.compressor_utils import (
get_dspark_swa_index_width,
)
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
_skip_unless_rope_quant()
torch.manual_seed(0)
device = "cuda"
block_size, window, draft = 64, 128, 4
width = get_dspark_swa_index_width(window, draft)
contexts = [context_len, context_len + 37]
num_reqs, num_tokens = len(contexts), len(contexts) * draft
num_rows = num_tokens + 2
pages = -(-(max(contexts) + draft) // block_size)
cache = torch.randn((num_reqs * pages, block_size, 512), device=device)
cache = cache.clamp_(-1, 1).to(torch.float8_e4m3fn)
attn = _make_rope_quant_attn(mod, num_rows, device)
attn.attn_sink = torch.full((128,), -float("inf"), device=device)
# As the DSpark SWA builder lays them out: the trailing window of context
# and the whole block, contiguous from column 0.
decode_indices = torch.full((num_tokens, width), -1, dtype=torch.int32)
visible, positions = [], []
for r, ctx in enumerate(contexts):
slots = torch.arange(max(ctx - window, 0), ctx + draft) + r * pages * block_size
for k in range(draft):
decode_indices[r * draft + k, : slots.numel()] = slots.to(torch.int32)
visible.append(slots)
positions.append(ctx + k)
query_start = torch.arange(0, num_tokens + 1, draft, dtype=torch.int32)
seq_lens = torch.tensor([c + draft for c in contexts], dtype=torch.int32)
metadata = DeepseekSparseSWAMetadata(
block_table=torch.arange(
num_reqs * pages, dtype=torch.int32, device=device
).view(num_reqs, pages),
slot_mapping=torch.zeros(num_tokens, dtype=torch.int64, device=device),
block_size=block_size,
seq_lens=seq_lens.to(device),
query_start_loc=query_start.to(device),
query_start_loc_cpu=query_start,
token_to_req_indices=torch.arange(
num_reqs, dtype=torch.int32, device=device
).repeat_interleave(draft),
decode_swa_indices=decode_indices.to(device),
decode_swa_lens=torch.tensor(
[v.numel() for v in visible], dtype=torch.int32, device=device
),
decode_swa_width=width,
is_valid_token=torch.ones(num_tokens, dtype=torch.bool, device=device),
num_decodes=num_reqs,
num_prefills=0,
num_decode_tokens=num_tokens,
num_prefill_tokens=0,
max_decode_query_len=draft,
)
q = torch.randn((num_rows, 128, 512), device=device).clamp_(-1, 1)
q = q.to(torch.float8_e4m3fn)
attention, z_unfused, z_fused = _rope_quant_vs_unfused(
attn, q, cache, metadata, torch.tensor(positions, device=device), num_tokens
)
keys_all = cache.flatten(0, 1).float()
for token, slots in enumerate(visible):
keys = keys_all[slots.to(device)]
weights = torch.softmax(q[token].float() @ keys.T * attn.scale, -1)
torch.testing.assert_close(
attention[token].float(), weights @ keys, atol=0.05, rtol=0.05
)
_assert_projects_alike(z_unfused, z_fused)
@@ -1,310 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""RoPE equivalence tests for the fused Kimi-K3 MLA epilogues."""
import pytest
import torch
from vllm.models.kimi_k3.nvidia.ops.fused_mla_key_concat_kv_cache import (
fused_mla_decode_q_concat_kv_cache_insert,
fused_mla_key_concat_ds_mla_insert,
fused_mla_key_concat_kv_cache_insert,
fused_mla_kv_concat,
fused_mla_kv_concat_quant_fp8,
fused_mla_qkv_quant_kv_cache_fp8_insert,
)
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda(), reason="Kimi-K3 fused MLA requires CUDA"
)
_DTYPE = torch.bfloat16
_NUM_TOKENS = 3
_NUM_HEADS = 4
_BLOCK_SIZE = 8
_POSITIONS = (1, 7, 13)
_SLOTS = (0, 3, 9)
def _randn(*shape: int, dtype: torch.dtype = _DTYPE) -> torch.Tensor:
return torch.randn(*shape, device="cuda", dtype=dtype) * 0.2
def _rope_cache(max_position: int = 32) -> torch.Tensor:
inv_freq = 1.0 / (
50000 ** (torch.arange(0, 64, 2, dtype=torch.float32, device="cuda") / 64)
)
positions = torch.arange(max_position, dtype=torch.float32, device="cuda")
freqs = torch.outer(positions, inv_freq)
# The fused epilogue reads the cos/sin table in fp32 (RoPE math runs in fp32).
return torch.cat((freqs.cos(), freqs.sin()), dim=-1)
def _apply_gptj_rope(
x: torch.Tensor, positions: torch.Tensor, cos_sin_cache: torch.Tensor
) -> torch.Tensor:
cos, sin = cos_sin_cache.index_select(0, positions).chunk(2, dim=-1)
for _ in range(x.ndim - 2):
cos = cos.unsqueeze(1)
sin = sin.unsqueeze(1)
x1 = x[..., ::2].float()
x2 = x[..., 1::2].float()
out1 = x1 * cos.float() - x2 * sin.float()
out2 = x2 * cos.float() + x1 * sin.float()
return torch.stack((out1, out2), dim=-1).flatten(-2).to(x.dtype)
def _cache_rows(cache: torch.Tensor, slots: torch.Tensor) -> torch.Tensor:
return cache.reshape(-1, cache.shape[-1]).index_select(0, slots)
def _assert_fp8_close(actual: torch.Tensor, expected: torch.Tensor) -> None:
torch.testing.assert_close(
actual.float(),
expected.to(torch.float8_e4m3fn).float(),
atol=0.03125,
rtol=0.15,
)
def _strided_context_inputs(
num_tokens: int, num_heads: int, dtype: torch.dtype, k_pe_fp8: bool
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""The real prefill-context layouts: ``k_nope``/``v`` as the two strided
halves of one kv_b_proj output and ``k_pe`` as a column slice of the gather
workspace (left in the fp8 cache layout for a plain fp8 cache)."""
kv_nope = _randn(num_tokens, num_heads, 256, dtype=dtype)
k_nope, v = kv_nope.split((128, 128), dim=-1)
workspace = _randn(num_tokens, 576, dtype=dtype)
if k_pe_fp8:
workspace = workspace.to(torch.float8_e4m3fn)
return k_nope, workspace[:, 512:].unsqueeze(1), v
@pytest.mark.parametrize("num_tokens", [0, _NUM_TOKENS])
@pytest.mark.parametrize("num_heads", [3, _NUM_HEADS])
@pytest.mark.parametrize("input_dtype", [torch.bfloat16, torch.float16])
@torch.inference_mode()
def test_context_kv_concat_accepts_strided_inputs(
num_tokens: int, num_heads: int, input_dtype: torch.dtype
) -> None:
k_nope, k_pe, _ = _strided_context_inputs(
num_tokens, num_heads, input_dtype, k_pe_fp8=False
)
k = fused_mla_kv_concat(k_nope, k_pe)
assert k.is_contiguous()
assert k.dtype == input_dtype
torch.testing.assert_close(k[..., :128], k_nope, atol=0, rtol=0)
torch.testing.assert_close(
k[..., 128:], k_pe.expand(-1, num_heads, -1), atol=0, rtol=0
)
@pytest.mark.parametrize("k_pe_dtype", ["input", "fp8"])
@pytest.mark.parametrize("num_tokens", [0, _NUM_TOKENS])
@pytest.mark.parametrize("num_heads", [3, _NUM_HEADS])
@pytest.mark.parametrize("input_dtype", [torch.bfloat16, torch.float16])
@torch.inference_mode()
def test_context_kv_pack_quantizes_strided_inputs(
k_pe_dtype: str,
num_tokens: int,
num_heads: int,
input_dtype: torch.dtype,
) -> None:
k_nope, k_pe, v = _strided_context_inputs(
num_tokens, num_heads, input_dtype, k_pe_fp8=k_pe_dtype == "fp8"
)
k_actual, v_actual = fused_mla_kv_concat_quant_fp8(k_nope, k_pe, v)
fp8 = torch.float8_e4m3fn
k_expected = torch.empty_like(k_actual)
k_expected[..., :128] = k_nope.to(fp8)
k_expected[..., 128:] = k_pe.to(fp8)
assert k_actual.is_contiguous()
assert v_actual.is_contiguous()
# The pack casts with the native pairwise converters, so it must be
# bit-identical to torch's `.to(fp8)` rather than merely close.
torch.testing.assert_close(k_actual.float(), k_expected.float(), atol=0, rtol=0)
torch.testing.assert_close(v_actual.float(), v.to(fp8).float(), atol=0, rtol=0)
@pytest.mark.parametrize("cache_kind", ["bf16", "fp8", "fp8_ds_mla"])
@torch.inference_mode()
def test_prefill_epilogue_fuses_gptj_rope(cache_kind: str) -> None:
torch.manual_seed(0)
positions = torch.tensor(_POSITIONS, device="cuda", dtype=torch.int64)
slots = torch.tensor(_SLOTS, device="cuda", dtype=torch.int64)
cos_sin_cache = _rope_cache()
q = _randn(_NUM_TOKENS, _NUM_HEADS, 192)
k_nope = _randn(_NUM_TOKENS, _NUM_HEADS, 128)
k_pe = _randn(_NUM_TOKENS, 64)
kv_c = _randn(_NUM_TOKENS, 512)
v = _randn(_NUM_TOKENS, _NUM_HEADS, 128)
q_expected = q.clone()
q_expected[..., 128:] = _apply_gptj_rope(
q_expected[..., 128:], positions, cos_sin_cache
)
k_pe_expected = _apply_gptj_rope(k_pe, positions, cos_sin_cache)
k_expected = torch.cat(
(k_nope, k_pe_expected[:, None, :].expand(-1, _NUM_HEADS, -1)), dim=-1
)
cache_expected = torch.cat((kv_c, k_pe_expected), dim=-1)
if cache_kind == "bf16":
cache = torch.zeros(2, _BLOCK_SIZE, 576, device="cuda", dtype=_DTYPE)
q_actual = q.clone()
k_actual = fused_mla_key_concat_kv_cache_insert(
q_actual,
k_nope,
k_pe,
kv_c,
cache,
slots,
positions,
cos_sin_cache,
)
torch.testing.assert_close(q_actual, q_expected)
torch.testing.assert_close(k_actual, k_expected)
torch.testing.assert_close(_cache_rows(cache, slots), cache_expected)
elif cache_kind == "fp8":
cache = torch.zeros(
2, _BLOCK_SIZE, 576, device="cuda", dtype=torch.float8_e4m3fn
)
one = torch.ones(1, device="cuda", dtype=torch.float32)
q_actual, k_actual, v_actual = fused_mla_qkv_quant_kv_cache_fp8_insert(
q,
k_nope,
k_pe,
kv_c,
v,
cache,
slots,
one,
one,
one,
one,
positions,
cos_sin_cache,
)
_assert_fp8_close(q_actual, q_expected)
_assert_fp8_close(k_actual, k_expected)
_assert_fp8_close(v_actual, v)
_assert_fp8_close(_cache_rows(cache, slots), cache_expected)
else:
cache = torch.zeros(2, _BLOCK_SIZE, 656, device="cuda", dtype=torch.uint8)
q_actual = q.clone()
k_actual = fused_mla_key_concat_ds_mla_insert(
q_actual,
k_nope,
k_pe,
kv_c,
cache,
slots,
positions,
cos_sin_cache,
)
rope_cache = _cache_rows(cache, slots)[:, 528:656].view(_DTYPE)
torch.testing.assert_close(q_actual, q_expected)
torch.testing.assert_close(k_actual, k_expected)
torch.testing.assert_close(rope_cache, k_pe_expected)
@pytest.mark.parametrize("cache_kind", ["bf16", "fp8", "fp8_ds_mla"])
@torch.inference_mode()
def test_decode_epilogue_fuses_gptj_rope(cache_kind: str) -> None:
torch.manual_seed(1)
positions = torch.tensor(_POSITIONS, device="cuda", dtype=torch.int64)
slots = torch.tensor(_SLOTS, device="cuda", dtype=torch.int64)
cos_sin_cache = _rope_cache()
ql_nope = _randn(_NUM_TOKENS, _NUM_HEADS, 512)
q_pe = _randn(_NUM_TOKENS, _NUM_HEADS, 64)
kv_c = _randn(_NUM_TOKENS, 512)
k_pe = _randn(_NUM_TOKENS, 64)
q_pe_expected = _apply_gptj_rope(q_pe, positions, cos_sin_cache)
k_pe_expected = _apply_gptj_rope(k_pe, positions, cos_sin_cache)
q_expected = torch.cat((ql_nope, q_pe_expected), dim=-1)
cache_expected = torch.cat((kv_c, k_pe_expected), dim=-1)
kwargs = {"positions": positions, "cos_sin_cache": cos_sin_cache}
if cache_kind == "bf16":
cache = torch.zeros(2, _BLOCK_SIZE, 576, device="cuda", dtype=_DTYPE)
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
ql_nope, q_pe, kv_c, k_pe, cache, slots, **kwargs
)
torch.testing.assert_close(q_actual, q_expected)
torch.testing.assert_close(_cache_rows(cache, slots), cache_expected)
elif cache_kind == "fp8":
cache = torch.zeros(
2, _BLOCK_SIZE, 576, device="cuda", dtype=torch.float8_e4m3fn
)
one = torch.ones(1, device="cuda", dtype=torch.float32)
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
ql_nope,
q_pe,
kv_c,
k_pe,
cache,
slots,
q_scale_inv=one,
cache_scale_inv=one,
**kwargs,
)
_assert_fp8_close(q_actual, q_expected)
_assert_fp8_close(_cache_rows(cache, slots), cache_expected)
else:
cache = torch.zeros(2, _BLOCK_SIZE, 656, device="cuda", dtype=torch.uint8)
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
ql_nope, q_pe, kv_c, k_pe, cache, slots, ds_mla=True, **kwargs
)
rope_cache = _cache_rows(cache, slots)[:, 528:656].view(_DTYPE)
torch.testing.assert_close(q_actual, q_expected)
torch.testing.assert_close(rope_cache, k_pe_expected)
@torch.inference_mode()
def test_decode_epilogue_preserves_nope_path() -> None:
torch.manual_seed(2)
slots = torch.tensor(_SLOTS, device="cuda", dtype=torch.int64)
ql_nope = _randn(_NUM_TOKENS, _NUM_HEADS, 512)
q_pe = _randn(_NUM_TOKENS, _NUM_HEADS, 64)
kv_c = _randn(_NUM_TOKENS, 512)
k_pe = _randn(_NUM_TOKENS, 64)
cache = torch.zeros(2, _BLOCK_SIZE, 576, device="cuda", dtype=_DTYPE)
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
ql_nope, q_pe, kv_c, k_pe, cache, slots
)
torch.testing.assert_close(q_actual, torch.cat((ql_nope, q_pe), dim=-1))
torch.testing.assert_close(
_cache_rows(cache, slots), torch.cat((kv_c, k_pe), dim=-1)
)
@torch.inference_mode()
@pytest.mark.parametrize("num_tokens", [64, 65])
def test_decode_epilogue_row_split_boundary(num_tokens: int) -> None:
"""Both sides of the M-based warps-per-row dispatch produce the same rows."""
torch.manual_seed(3)
num_blocks = (num_tokens + _BLOCK_SIZE - 1) // _BLOCK_SIZE
slots = torch.randperm(num_blocks * _BLOCK_SIZE, device="cuda")[:num_tokens].long()
ql_nope = _randn(num_tokens, _NUM_HEADS, 512)
q_pe = _randn(num_tokens, _NUM_HEADS, 64)
kv_c = _randn(num_tokens, 512)
k_pe = _randn(num_tokens, 64)
cache = torch.zeros(num_blocks, _BLOCK_SIZE, 576, device="cuda", dtype=_DTYPE)
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
ql_nope, q_pe, kv_c, k_pe, cache, slots
)
torch.testing.assert_close(q_actual, torch.cat((ql_nope, q_pe), dim=-1))
torch.testing.assert_close(
_cache_rows(cache, slots), torch.cat((kv_c, k_pe), dim=-1)
)
@@ -1,257 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import vllm._custom_ops as ops
from vllm.models.kimi_k3.nvidia.ops.fused_mla_key_concat_kv_cache import (
fused_mla_decode_q_concat_kv_cache_insert,
fused_mla_key_concat_ds_mla_insert,
fused_mla_key_concat_kv_cache_insert,
fused_mla_qkv_quant_kv_cache_fp8_insert,
)
from vllm.platforms import current_platform
NUM_TOKENS = 4
NUM_HEADS = 2
KV_LORA_RANK = 512
NOPE_HEAD_DIM = 128
ROPE_HEAD_DIM = 64
QK_HEAD_DIM = NOPE_HEAD_DIM + ROPE_HEAD_DIM
V_HEAD_DIM = 128
CACHE_ENTRY = KV_LORA_RANK + ROPE_HEAD_DIM
DS_MLA_CACHE_ENTRY = 656
BLOCK_SIZE = 8
OWNED_TOKEN_INDICES = [1, 3]
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda(), reason="Kimi-K3 fused MLA ops require CUDA"
)
def _inputs() -> dict[str, torch.Tensor]:
torch.manual_seed(0)
def make(*shape: int) -> torch.Tensor:
return torch.randn(*shape, device="cuda", dtype=torch.bfloat16)
return {
"q": make(NUM_TOKENS, NUM_HEADS, QK_HEAD_DIM),
"k_nope": make(NUM_TOKENS, NUM_HEADS, NOPE_HEAD_DIM),
"k_pe": make(NUM_TOKENS, ROPE_HEAD_DIM),
"kv_c": make(NUM_TOKENS, KV_LORA_RANK),
"v": make(NUM_TOKENS, NUM_HEADS, V_HEAD_DIM),
"ql_nope": make(NUM_TOKENS, NUM_HEADS, KV_LORA_RANK),
"q_pe": make(NUM_TOKENS, NUM_HEADS, ROPE_HEAD_DIM),
}
def _slot_mapping() -> torch.Tensor:
return torch.tensor([-1, 0, -1, 5], device="cuda", dtype=torch.int64)
def _owned(tensor: torch.Tensor) -> torch.Tensor:
return tensor[OWNED_TOKEN_INDICES]
def _full_key(k_nope: torch.Tensor, k_pe: torch.Tensor) -> torch.Tensor:
shared_pe = k_pe[:, None, :].expand(-1, k_nope.shape[1], -1)
return torch.cat((k_nope, shared_pe), dim=-1)
def _latent_query(ql_nope: torch.Tensor, q_pe: torch.Tensor) -> torch.Tensor:
return torch.cat((ql_nope, q_pe), dim=-1)
def _make_cache(cache_format: str) -> torch.Tensor:
if cache_format == "bf16":
return torch.full(
(1, BLOCK_SIZE, CACHE_ENTRY),
-7,
device="cuda",
dtype=torch.bfloat16,
)
if cache_format == "fp8":
return torch.full(
(1, BLOCK_SIZE, CACHE_ENTRY),
1.0,
device="cuda",
dtype=torch.float32,
).to(torch.float8_e4m3fn)
assert cache_format == "fp8_ds_mla"
return torch.full(
(1, BLOCK_SIZE, DS_MLA_CACHE_ENTRY),
165,
device="cuda",
dtype=torch.uint8,
)
def _reference_cache(
cache_format: str,
inputs: dict[str, torch.Tensor],
initial_cache: torch.Tensor,
slots: torch.Tensor,
) -> torch.Tensor:
reference_cache = initial_cache.clone()
valid_slots = _owned(slots)
if cache_format == "fp8_ds_mla":
ops.concat_and_cache_mla(
_owned(inputs["kv_c"]),
_owned(inputs["k_pe"]),
reference_cache,
valid_slots,
cache_format,
torch.ones(1, device="cuda", dtype=torch.float32),
)
return reference_cache
latent = torch.cat(
(_owned(inputs["kv_c"]), _owned(inputs["k_pe"])),
dim=-1,
)
flat_reference_cache = reference_cache.view(-1, CACHE_ENTRY)
for row, slot in zip(latent, valid_slots.tolist()):
flat_reference_cache[slot].copy_(row.to(reference_cache.dtype))
return reference_cache
def _assert_cache_matches_reference(
cache: torch.Tensor,
reference_cache: torch.Tensor,
initial_cache: torch.Tensor,
) -> None:
assert torch.equal(cache, reference_cache)
for slot in (0, 5):
assert not torch.equal(cache[0, slot], initial_cache[0, slot])
for slot in (1, 2, 3, 4, 6, 7):
assert torch.equal(cache[0, slot], initial_cache[0, slot])
@pytest.mark.parametrize("cache_format", ["bf16", "fp8_ds_mla"])
def test_prefill_concat_ignores_negative_slots_for_cache(cache_format: str) -> None:
inputs = _inputs()
slots = _slot_mapping()
initial_cache = _make_cache(cache_format)
mixed_cache = initial_cache.clone()
reference_cache = _reference_cache(cache_format, inputs, initial_cache, slots)
op = (
fused_mla_key_concat_kv_cache_insert
if cache_format == "bf16"
else fused_mla_key_concat_ds_mla_insert
)
output = op(
inputs["q"],
inputs["k_nope"],
inputs["k_pe"],
inputs["kv_c"],
mixed_cache,
slots,
)
torch.testing.assert_close(
output, _full_key(inputs["k_nope"], inputs["k_pe"]), rtol=0, atol=0
)
_assert_cache_matches_reference(mixed_cache, reference_cache, initial_cache)
def test_fp8_prefill_outputs_ignore_negative_slots_for_cache() -> None:
inputs = _inputs()
slots = _slot_mapping()
initial_cache = _make_cache("fp8")
mixed_cache = initial_cache.clone()
reference_cache = _reference_cache("fp8", inputs, initial_cache, slots)
scale_inv = torch.ones(1, device="cuda", dtype=torch.float32)
q, k, v = fused_mla_qkv_quant_kv_cache_fp8_insert(
inputs["q"],
inputs["k_nope"],
inputs["k_pe"],
inputs["kv_c"],
inputs["v"],
mixed_cache,
slots,
scale_inv,
scale_inv,
scale_inv,
scale_inv,
)
expected = (
inputs["q"].to(torch.float8_e4m3fn),
_full_key(inputs["k_nope"], inputs["k_pe"]).to(torch.float8_e4m3fn),
inputs["v"].to(torch.float8_e4m3fn),
)
for actual, reference in zip((q, k, v), expected):
torch.testing.assert_close(actual.float(), reference.float(), rtol=0, atol=0)
_assert_cache_matches_reference(mixed_cache, reference_cache, initial_cache)
@pytest.mark.parametrize("cache_format", ["bf16", "fp8", "fp8_ds_mla"])
def test_decode_concat_ignores_negative_slots_for_cache(cache_format: str) -> None:
inputs = _inputs()
slots = _slot_mapping()
initial_cache = _make_cache(cache_format)
mixed_cache = initial_cache.clone()
reference_cache = _reference_cache(cache_format, inputs, initial_cache, slots)
scale_inv = torch.ones(1, device="cuda", dtype=torch.float32)
output = fused_mla_decode_q_concat_kv_cache_insert(
inputs["ql_nope"],
inputs["q_pe"],
inputs["kv_c"],
inputs["k_pe"],
mixed_cache,
slots,
ds_mla=cache_format == "fp8_ds_mla",
q_scale_inv=scale_inv if cache_format == "fp8" else None,
cache_scale_inv=scale_inv if cache_format == "fp8" else None,
)
expected = _latent_query(inputs["ql_nope"], inputs["q_pe"])
if cache_format == "fp8":
expected = expected.to(torch.float8_e4m3fn)
torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0)
else:
torch.testing.assert_close(output, expected, rtol=0, atol=0)
_assert_cache_matches_reference(mixed_cache, reference_cache, initial_cache)
@pytest.mark.parametrize("decode", [False, True])
def test_ds_mla_cache_insert_bit_compatible_with_reference(decode: bool) -> None:
"""Fused ds_mla insertion must match the reference bit-for-bit."""
torch.manual_seed(0)
num_tokens, num_blocks = 33, 16
dt = torch.bfloat16
kv_c = torch.randn(num_tokens, KV_LORA_RANK, device="cuda", dtype=dt)
k_pe = torch.randn(num_tokens, 1, ROPE_HEAD_DIM, device="cuda", dtype=dt)
k_nope = torch.randn(num_tokens, NUM_HEADS, NOPE_HEAD_DIM, device="cuda", dtype=dt)
q = torch.randn(num_tokens, NUM_HEADS, QK_HEAD_DIM, device="cuda", dtype=dt)
slots = torch.randperm(num_blocks * BLOCK_SIZE, device="cuda")[:num_tokens].long()
slots[torch.rand(num_tokens, device="cuda") < 0.3] = -1
ref = torch.full(
(num_blocks, BLOCK_SIZE, DS_MLA_CACHE_ENTRY),
165,
device="cuda",
dtype=torch.uint8,
)
got = ref.clone()
scale = torch.ones(1, device="cuda", dtype=torch.float32)
ops.concat_and_cache_mla(kv_c, k_pe.squeeze(1), ref, slots, "fp8_ds_mla", scale)
if decode:
ql_nope = torch.randn(
num_tokens, NUM_HEADS, KV_LORA_RANK, device="cuda", dtype=dt
)
fused_mla_decode_q_concat_kv_cache_insert(
ql_nope, q[..., -ROPE_HEAD_DIM:], kv_c, k_pe, got, slots, ds_mla=True
)
else:
fused_mla_key_concat_ds_mla_insert(q, k_nope, k_pe, kv_c, got, slots)
assert torch.equal(ref, got)
rows = got.view(-1, DS_MLA_CACHE_ENTRY)[slots[slots >= 0]]
scales = rows[:, KV_LORA_RANK : KV_LORA_RANK + 16].view(torch.float32)
torch.testing.assert_close(
torch.log2(scales), torch.log2(scales).round(), rtol=0, atol=0
)
File diff suppressed because it is too large Load Diff
@@ -1,143 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""fp8 query and index-K cache on the Triton indexer must agree with bf16.
The MiniMax M3 lightning-indexer scores only feed a top-k block ranking (the
attention itself reads the main KV cache), so an fp8 (e4m3) side cache trades
~2% relative score error for half the per-step index-K read bandwidth — the
dominant linear-in-context decode cost at long context. #45892 enabled fp8
index caches on the SM100 MSA path; this covers the Triton impl used
everywhere else (e.g. SM120), for both the decode and prefill scorers.
"""
import pytest
import torch
from vllm.models.minimax_m3.common.ops.index_topk import (
SPARSE_BLOCK_SIZE,
minimax_m3_index_decode,
minimax_m3_index_score,
minimax_m3_index_topk,
)
from vllm.platforms import current_platform
FP8_CUDA_SUPPORTED = current_platform.is_cuda() and current_platform.supports_fp8()
PAGE = SPARSE_BLOCK_SIZE
HEAD_DIM = 128
HEADS = 1
TOPK = 16
NUM_PAGES = 100
SEQ_LEN = 96 * PAGE + 37
PLANTED = (7, 40, 88)
def _make_inputs(device: str) -> tuple[torch.Tensor, torch.Tensor]:
torch.manual_seed(3)
keys = torch.randn(NUM_PAGES * PAGE, HEAD_DIM, dtype=torch.bfloat16, device=device)
# RMS-normalize like the real (post index_k_norm) keys.
keys = keys * keys.float().pow(2).mean(-1, keepdim=True).add(1e-6).rsqrt().to(
torch.bfloat16
)
q = torch.randn(1, HEADS, HEAD_DIM, dtype=torch.bfloat16, device=device)
for b in PLANTED:
keys[b * PAGE + 5] = (q[0, 0] * 3).to(torch.bfloat16)
return q, keys
def _check_selection(top_bf16: set[int], top_fp8: set[int]) -> None:
# Planted (clearly relevant) blocks must never be lost.
assert all(b in top_fp8 for b in PLANTED)
# Selection may differ only at the noise floor (near-tied filler blocks).
assert len(top_bf16 & top_fp8) >= TOPK - 2
@pytest.mark.skipif(not FP8_CUDA_SUPPORTED, reason="CUDA FP8 support required")
@torch.inference_mode()
def test_fp8_index_query_and_cache_decode_topk_matches_bf16() -> None:
dev = "cuda"
q, keys = _make_inputs(dev)
block_table = torch.arange(NUM_PAGES, dtype=torch.int32, device=dev).unsqueeze(0)
seq_lens = torch.tensor([SEQ_LEN], dtype=torch.int32, device=dev)
def select(cache: torch.Tensor) -> set[int]:
query = q.to(cache.dtype)
idx = minimax_m3_index_decode(
query,
cache.view(NUM_PAGES, PAGE, HEAD_DIM).contiguous(),
block_table,
seq_lens,
SEQ_LEN,
TOPK,
0,
0,
HEADS,
1,
1,
)
return {int(b) for b in idx[0, 0].tolist() if b >= 0}
_check_selection(select(keys), select(keys.to(torch.float8_e4m3fn)))
@pytest.mark.skipif(not FP8_CUDA_SUPPORTED, reason="CUDA FP8 support required")
@torch.inference_mode()
def test_fp8_index_query_and_cache_prefill_topk_matches_bf16() -> None:
dev = "cuda"
q, keys = _make_inputs(dev)
block_table = torch.arange(NUM_PAGES, dtype=torch.int32, device=dev).unsqueeze(0)
seq_lens = torch.tensor([SEQ_LEN], dtype=torch.int32, device=dev)
# single decode-like query token at the end of the sequence
cu_seqlens_q = torch.tensor([0, 1], dtype=torch.int32, device=dev)
prefix_lens = torch.tensor([SEQ_LEN - 1], dtype=torch.int32, device=dev)
def select(cache: torch.Tensor) -> set[int]:
query = q.to(cache.dtype)
score = minimax_m3_index_score(
query,
cache.view(NUM_PAGES, PAGE, HEAD_DIM).contiguous(),
block_table,
cu_seqlens_q,
seq_lens,
prefix_lens,
1,
SEQ_LEN,
HEADS,
)
idx = minimax_m3_index_topk(score, cu_seqlens_q, prefix_lens, 1, TOPK, 0, 0)
return {int(b) for b in idx[0, 0].tolist() if b >= 0}
_check_selection(select(keys), select(keys.to(torch.float8_e4m3fn)))
@pytest.mark.skipif(not FP8_CUDA_SUPPORTED, reason="CUDA FP8 support required")
@torch.inference_mode()
def test_bf16_query_against_fp8_cache_decode_topk_matches_bf16() -> None:
"""A mixed pair must upcast rather than emit a mixed-dtype fp8 dot.
Production allocates index_q with the index-K cache dtype, so the scorers
normally see a matched pair and keep the native FP8 MMA. This pins the
other branch: a bf16 query against an e4m3 cache still compiles and still
ranks the planted blocks.
"""
dev = "cuda"
q, keys = _make_inputs(dev)
block_table = torch.arange(NUM_PAGES, dtype=torch.int32, device=dev).unsqueeze(0)
seq_lens = torch.tensor([SEQ_LEN], dtype=torch.int32, device=dev)
def select(cache: torch.Tensor) -> set[int]:
idx = minimax_m3_index_decode(
q, # bf16 query, deliberately not cast to the cache dtype
cache.view(NUM_PAGES, PAGE, HEAD_DIM).contiguous(),
block_table,
seq_lens,
SEQ_LEN,
TOPK,
0,
0,
HEADS,
1,
1,
)
return {int(b) for b in idx[0, 0].tolist() if b >= 0}
_check_selection(select(keys), select(keys.to(torch.float8_e4m3fn)))
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -1,226 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm correctness tests for the strided DSA candidate mask.
``_apply_candidate_mask_strided`` is a decode-path variant of the shared
``apply_candidate_mask``. It differs in two ways that need pinning down:
* it launches a fixed number of programs that stride the column axis, rather
than one program per 1024-column tile, so at ``max_model_len`` widths the
grid no longer scales with a workspace that is mostly padding;
* it stops at each row's ``end`` instead of sanitizing the whole width.
The second is only sound because every consumer bounds its scan by the same
ends -- ``top_k_per_row_decode`` takes ``rowEnd`` from ``seq_lens``. So the
contract is agreement with the shared kernel over ``[0, end)`` and nothing
beyond it, which is what these tests assert rather than comparing whole rows.
"""
import pytest
import torch
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(), reason="ROCm-specific tests"
)
BLOCK_SIZE = 8
TOPK_BLOCKS = 64
# Above 128 tiles the grid saturates and programs start striding; below it the
# min() clamp applies instead. Both regimes need covering, so the widths here
# deliberately straddle 128 * 1024.
NARROW = 8192
WIDE = 262144
def _inputs(rows, width, ends, starts, *, block_size, col_stride, dtype):
"""Logits, bounds and candidate blocks shared by both kernels."""
torch.manual_seed(0)
# A column stride > 1 keeps the kernels honest about stride_col; the decode
# caller slices a workspace, so contiguous rows are not guaranteed.
logits = torch.randn(rows, width * col_stride, device="cuda")[:, ::col_stride]
logits[0, : min(16, width)] = 0.0
if rows > 2:
logits[2, 5] = float("nan")
row_ke = torch.tensor(ends, device="cuda", dtype=dtype)
row_ks = torch.tensor(starts, device="cuda", dtype=dtype) if starts else None
nblocks = (width + block_size - 1) // block_size
candidates = torch.randint(
0, nblocks, (rows, TOPK_BLOCKS), device="cuda", dtype=torch.int32
)
# -1 is the "no block" sentinel the selector emits for short rows.
candidates[:, ::7] = -1
return logits, row_ks, row_ke, candidates
def _run_both(logits, row_ks, row_ke, candidates, block_size, row_repeat):
from vllm.model_executor.kernels.attention.dsa.candidate_blocks import (
apply_candidate_mask,
)
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_apply_candidate_mask_strided,
)
ref = logits.clone()
got = logits.clone()
apply_candidate_mask(ref, row_ks, row_ke, candidates, block_size, row_repeat)
_apply_candidate_mask_strided(
got, row_ks, row_ke, candidates, block_size, row_repeat
)
return ref, got
def _assert_agrees_within_ends(ref, got, row_ke, row_repeat):
"""Compare only the region a consumer can read."""
for row in range(ref.shape[0]):
end = int(row_ke[row // row_repeat])
if end <= 0:
continue
torch.testing.assert_close(
got[row, :end],
ref[row, :end],
rtol=0,
atol=0,
equal_nan=True,
msg=lambda m, r=row, e=end: f"row {r} differs within [0, {e}): {m}",
)
@pytest.mark.parametrize("width", [NARROW, WIDE])
@pytest.mark.parametrize("col_stride", [1, 2])
def test_matches_shared_kernel_ragged_ends(width, col_stride):
"""Ends on and around tile boundaries, plus the empty and full rows."""
ends = [0, 1, 1023, 1024, 1025, width - 1, width, width // 2]
logits, row_ks, row_ke, candidates = _inputs(
len(ends),
width,
ends,
None,
block_size=BLOCK_SIZE,
col_stride=col_stride,
dtype=torch.int32,
)
ref, got = _run_both(logits, row_ks, row_ke, candidates, BLOCK_SIZE, 1)
_assert_agrees_within_ends(ref, got, row_ke, 1)
@pytest.mark.parametrize("block_size", [8, 16])
def test_matches_shared_kernel_with_starts(block_size):
"""Non-zero starts: the decode caller passes zeros, prefill does not."""
ends = [0, 64, 4096, 8192]
starts = [0, 7, 1024, 3]
logits, row_ks, row_ke, candidates = _inputs(
len(ends),
NARROW,
ends,
starts,
block_size=block_size,
col_stride=1,
dtype=torch.int64,
)
ref, got = _run_both(logits, row_ks, row_ke, candidates, block_size, 1)
_assert_agrees_within_ends(ref, got, row_ke, 1)
def test_matches_shared_kernel_row_repeat():
"""Speculative decode: next_n logit rows share one bound."""
row_repeat = 3
ends = [0, 1025, WIDE]
logits, row_ks, row_ke, candidates = _inputs(
len(ends) * row_repeat,
WIDE,
ends,
None,
block_size=BLOCK_SIZE,
col_stride=1,
dtype=torch.int32,
)
ref, got = _run_both(logits, row_ks, row_ke, candidates, BLOCK_SIZE, row_repeat)
_assert_agrees_within_ends(ref, got, row_ke, row_repeat)
def test_leaves_columns_past_end_untouched():
"""The deliberate difference from the shared kernel, pinned explicitly.
Skipping the tail is where the speedup comes from, so a change that
quietly restores full-width sanitizing should fail here rather than just
get slower. Work stops at the tile boundary containing ``end``, not at
``end`` itself, so that is the bound asserted.
"""
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_MASK_TILE,
_apply_candidate_mask_strided,
)
ends = [0, 1, 1025, WIDE // 2]
logits, row_ks, row_ke, candidates = _inputs(
len(ends),
WIDE,
ends,
None,
block_size=BLOCK_SIZE,
col_stride=1,
dtype=torch.int32,
)
sentinel = 1.5
logits.fill_(sentinel)
_apply_candidate_mask_strided(logits, row_ks, row_ke, candidates, BLOCK_SIZE, 1)
for row, end in enumerate(ends):
touched = min(-(-end // _MASK_TILE) * _MASK_TILE, WIDE)
tail = logits[row, touched:]
assert torch.equal(tail, torch.full_like(tail, sentinel)), (
f"row {row} (end {end}) was written past column {touched}"
)
# And the bound is tight: a row that ends early must not have cost a
# full-width pass, which is the whole point of the kernel.
assert touched <= end + _MASK_TILE
def test_cudagraph_replay_tracks_changing_ends():
"""The grid is static so a FULL capture stays valid when ends change.
Capturing bakes the grid, so a data-dependent program count would freeze
whatever lengths were resident at capture. Replaying against longer and
shorter contexts than the captured ones is the property that matters.
"""
from vllm.model_executor.kernels.attention.dsa.candidate_blocks import (
apply_candidate_mask,
)
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_apply_candidate_mask_strided,
)
rows = 4
captured_ends = [1024, 4096, 8192, 2048]
logits, row_ks, row_ke, candidates = _inputs(
rows,
WIDE,
captured_ends,
None,
block_size=BLOCK_SIZE,
col_stride=1,
dtype=torch.int32,
)
base = logits.clone()
# Warm up outside the capture: Triton compiles on first call, and autotune
# launches would otherwise happen inside the graph.
_apply_candidate_mask_strided(
logits.clone(), row_ks, row_ke, candidates, BLOCK_SIZE, 1
)
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
_apply_candidate_mask_strided(logits, row_ks, row_ke, candidates, BLOCK_SIZE, 1)
for replay_ends in ([1, 131072, WIDE, 0], [WIDE, 1025, 3, 65536]):
row_ke.copy_(torch.tensor(replay_ends, device="cuda", dtype=row_ke.dtype))
logits.copy_(base)
graph.replay()
torch.accelerator.synchronize()
ref = base.clone()
apply_candidate_mask(ref, row_ks, row_ke, candidates, BLOCK_SIZE, 1)
_assert_agrees_within_ends(ref, logits, row_ke, 1)
File diff suppressed because it is too large Load Diff
@@ -1,262 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression test for causal masking in the AITER MLA verify MTP path.
When ``num_heads < 16`` the ROCm AITER MLA backend cannot use the ASM decode
kernel for a multi-token block, so ``AiterMLAImpl.forward_mqa`` routes verify
through ``mla_gluon``'s native 4-D MTP entry (``q`` shaped
``[batch, qlen, nhead, dim]`` with ``use_2d_view=False``). aiter applies the
causal tail internally: position ``t`` attends KV ``[0, seq_len - qlen + t]``.
The test drives the real builder and ``forward_mqa`` over a multi-token decode
batch, intercepts the arguments actually handed to the Gluon kernel, and pins
that the 4-D MTP contract is used with per-request ``block_table`` and
``cache_seqlens`` rather than a flattened per-token paged-KV view.
"""
import types
from unittest.mock import patch
import pytest
import torch
from vllm._aiter_ops import is_aiter_found
from vllm.platforms import current_platform
def _on_rocm_with_aiter() -> bool:
return current_platform.is_rocm() and is_aiter_found()
# The production verify MTP path is gfx950-only. This test replaces the Gluon
# kernel with a spy and forces the architecture feature probe so the metadata
# regression remains testable on every ROCm AITER CI runner.
pytestmark = pytest.mark.skipif(
not _on_rocm_with_aiter(),
reason="ROCM_AITER_MLA verify MTP requires ROCm and AITER",
)
# Below the 16-head threshold that selects the MTP path. DeepSeek-V3 / R1 at
# TP=16 and Kimi-K3 at TP=8 both land here.
NUM_QUERY_HEADS = 12
KV_LORA_RANK = 512
QK_NOPE_HEAD_DIM = 128
QK_ROPE_HEAD_DIM = 64
V_HEAD_DIM = 128
HEAD_SIZE = KV_LORA_RANK + QK_ROPE_HEAD_DIM
# 1 + num_speculative_tokens, i.e. the verify block length.
QLEN = 8
# Committed context per request, including a fresh request at zero context
# where the whole KV range is the verify block itself.
CONTEXT_LENS = [0, 1, 37, 512]
# A cudagraph padding request: seq_len 0.
PADDING_ROWS = 1
# The Gluon path flattens the KV cache to one page per token.
PAGE_SIZE = 1
MAX_MODEL_LEN = 1024
MAX_NUM_SEQS = 8
# The builder resolves the verify routing and records it on the metadata, so
# the gfx950 feature probe has to be forced for the build as well as for the
# forward; ROCm CI also runs gfx942, where the real probe is False.
_GLUON_SUPPORTED_TARGET = (
"vllm.v1.attention.backends.mla.rocm_aiter_mla._gluon_mla_decode_supported"
)
def _seq_lens() -> list[int]:
return [c + QLEN for c in CONTEXT_LENS] + [0] * PADDING_ROWS
def _run_verify_block():
"""Drive the real builder + forward_mqa over one multi-token verify block.
Returns ``(metadata, captured)`` where ``captured`` holds the Gluon kwargs.
"""
from tests.v1.attention.utils import (
BatchSpec,
create_common_attn_metadata,
create_vllm_config,
)
from vllm.config import SpeculativeConfig
from vllm.config.vllm import set_current_vllm_config
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAImpl
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.kv_cache_interface import MLAAttentionSpec
from vllm.v1.worker.workspace import init_workspace_manager
device = torch.device("cuda:0")
seq_lens = _seq_lens()
num_reqs = len(seq_lens)
# One flat page per token, arange block indices, plus room to spare.
num_gpu_blocks = num_reqs * max(seq_lens) + 256
vllm_config = create_vllm_config(
model_name="deepseek-ai/DeepSeek-R1",
max_model_len=MAX_MODEL_LEN,
num_gpu_blocks=num_gpu_blocks,
block_size=PAGE_SIZE,
max_num_seqs=MAX_NUM_SEQS,
max_num_batched_tokens=8192,
hf_config_override={"num_attention_heads": NUM_QUERY_HEADS},
)
# What raises reorder_batch_threshold to QLEN, so a QLEN-token block is
# classified as a decode instead of a prefill. ngram needs no draft model.
vllm_config.speculative_config = SpeculativeConfig(
method="ngram", num_speculative_tokens=QLEN - 1
)
vllm_config.model_config.get_num_attention_heads = types.MethodType( # type: ignore[method-assign]
lambda self, parallel_config, arch_config=None: NUM_QUERY_HEADS,
vllm_config.model_config,
)
spec = MLAAttentionSpec(
block_size=PAGE_SIZE,
num_kv_heads=1,
head_size=vllm_config.model_config.get_head_size(),
dtype=vllm_config.model_config.dtype,
cache_dtype_str="auto",
)
backend_cls = AttentionBackendEnum.ROCM_AITER_MLA.get_class()
builder_cls = backend_cls.get_builder_cls()
impl_cls = backend_cls.get_impl_cls()
# The builder reads layer.prefill_backend from static_forward_context; a
# stub with the attribute is enough for metadata construction.
layer_name = "placeholder"
vllm_config.compilation_config.static_forward_context[layer_name] = (
types.SimpleNamespace(
prefill_backend=torch.empty((1,)),
q_lora_rank=None,
kv_lora_rank=KV_LORA_RANK,
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
v_head_dim=V_HEAD_DIM,
)
)
init_workspace_manager(device)
batch_spec = BatchSpec(seq_lens=seq_lens, query_lens=[QLEN] * num_reqs)
captured: dict = {}
def spy(**kwargs):
captured.update(kwargs)
captured["q_nope"] = kwargs["q_nope"].detach().clone()
captured["page_table"] = kwargs["page_table"].detach().clone()
captured["seq_info"] = kwargs["seq_info"].detach().clone()
with (
patch(_GLUON_SUPPORTED_TARGET, lambda: True),
set_current_vllm_config(vllm_config),
):
builder = builder_cls(spec, [layer_name], vllm_config, device)
common_attn_metadata = create_common_attn_metadata(
batch_spec, PAGE_SIZE, device, arange_block_indices=True
)
metadata = builder.build(
common_prefix_len=0, common_attn_metadata=common_attn_metadata
)
impl = impl_cls(
num_heads=NUM_QUERY_HEADS,
head_size=HEAD_SIZE,
scale=(QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM) ** -0.5,
num_kv_heads=1,
alibi_slopes=None,
sliding_window=None,
kv_cache_dtype="auto",
logits_soft_cap=None,
attn_type="decoder",
kv_sharing_target_layer_name=None,
q_lora_rank=None,
kv_lora_rank=KV_LORA_RANK,
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
qk_head_dim=QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM,
v_head_dim=V_HEAD_DIM,
kv_b_proj=None,
)
num_rows = num_reqs * QLEN
dtype = torch.bfloat16
q_nope = torch.zeros(
num_rows, NUM_QUERY_HEADS, KV_LORA_RANK, dtype=dtype, device=device
)
q_pe = torch.zeros(
num_rows, NUM_QUERY_HEADS, QK_ROPE_HEAD_DIM, dtype=dtype, device=device
)
kv_cache = torch.zeros(
num_gpu_blocks, PAGE_SIZE, HEAD_SIZE, dtype=dtype, device=device
)
# The Gluon kernel only reads the metadata this test is about, so a spy in
# its place keeps the assertions independent of the AITER build.
with (
patch(_GLUON_SUPPORTED_TARGET, lambda: True),
patch(
"vllm.v1.attention.backends.mla.rocm_aiter_mla._get_mla_gluon",
lambda: spy,
),
):
assert isinstance(impl, AiterMLAImpl)
impl.forward_mqa((q_nope, q_pe), kv_cache, metadata, layer=None)
return metadata, captured
def test_verify_mtp_uses_native_4d_gluon_entry():
"""Verify uses 4-D q with per-request paged_kv metadata.
Regression guard: before the fix every flattened row of a request got that
request's whole paged-KV range, so verify position t attended to the draft
tokens after it. The native MTP path delegates causal masking to aiter.
"""
metadata, captured = _run_verify_block()
decode = metadata.decode
assert decode is not None, "batch was not classified as a decode"
assert decode.max_qo_len == QLEN, (
f"expected a {QLEN}-token verify block, got max_qo_len="
f"{decode.max_qo_len}; the MTP path under test was not reached"
)
assert decode.use_gluon_verify, (
"the builder owns verify routing, so the metadata -- not the impl -- "
"has to select the Gluon MTP entry for a small-head bf16 verify block"
)
assert captured, "forward_mqa did not reach the Gluon kernel"
seq_lens = _seq_lens()
num_reqs = len(seq_lens)
q_nope = captured["q_nope"]
assert q_nope.dim() == 4, (
f"verify must pass 4-D q to mla_gluon, got {q_nope.dim()}-D shape "
f"{tuple(q_nope.shape)}"
)
assert q_nope.shape[:2] == (num_reqs, QLEN), (
f"expected q shape [num_reqs={num_reqs}, qlen={QLEN}, ...], "
f"got {tuple(q_nope.shape)}"
)
assert q_nope.shape[2] == NUM_QUERY_HEADS
assert captured["use_2d_view"] is False
page_table = captured["page_table"]
seq_info = captured["seq_info"]
assert page_table.dim() == 1, (
f"verify MTP passes the 1-D paged_kv_indices buffer, got {page_table.dim()}-D"
)
assert seq_info.dim() == 1 and seq_info.numel() == num_reqs + 1, (
f"verify MTP passes paged_kv_indptr [num_reqs + 1], got shape "
f"{tuple(seq_info.shape)}"
)
assert decode is not None and decode.paged_kv_indptr is not None
assert torch.equal(seq_info, decode.paged_kv_indptr[: num_reqs + 1])
assert torch.equal(
page_table[: int(seq_info[-1].item())],
decode.paged_kv_indices[: int(seq_info[-1].item())],
)
@@ -1,225 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm kernel correctness tests for AITER MLA decode.
Compares ``rocm_aiter_ops.mla_decode_fwd`` against a pure PyTorch reference
under the absorbed MLA formulation (DeepSeek-V3/V4) with varied batch sizes,
head counts, and sequence lengths.
"""
import pytest
import torch
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
_SKIP_NON_MI3XX = True
if current_platform.is_rocm():
from vllm.platforms.rocm import on_mi3xx
_SKIP_NON_MI3XX = not on_mi3xx()
pytestmark = [
pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-specific tests"),
pytest.mark.skipif(_SKIP_NON_MI3XX, reason="MI300/MI350 ROCm only"),
]
# DeepSeek-V3/V4 MLA config.
Q_HEAD_DIM = 576 # kv_lora_rank + qk_rope_head_dim
V_HEAD_DIM = 512 # kv_lora_rank
SM_SCALE = Q_HEAD_DIM**-0.5
NUM_HEADS = [16, 128]
BATCH_SIZES = [1, 4, 16]
KV_SEQ_LENS = [16, 256]
ATOL, RTOL = 1.5e-2, 1e-2
def _require_aiter():
from vllm._aiter_ops import is_aiter_found_and_supported
if not is_aiter_found_and_supported():
pytest.skip("aiter is required on supported ROCm hardware for this test")
def _ref_mla_decode(
q: torch.Tensor,
kv_buffer: torch.Tensor,
kv_indptr: torch.Tensor,
kv_indices: torch.Tensor,
) -> torch.Tensor:
"""Pure PyTorch reference for absorbed MLA decode.
K uses full q_head_dim (576) for scoring, V uses first v_head_dim (512)
for output. This mirrors the production kernel's absorbed formulation.
"""
batch_size, num_heads, _ = q.shape
output = torch.zeros(
batch_size, num_heads, V_HEAD_DIM, dtype=q.dtype, device=q.device
)
for b in range(batch_size):
start = kv_indptr[b].item()
end = kv_indptr[b + 1].item()
token_indices = kv_indices[start:end]
k = kv_buffer[token_indices].float()
v = kv_buffer[token_indices, :V_HEAD_DIM].float()
for h in range(num_heads):
q_h = q[b, h, :].float()
scores = torch.mv(k, q_h) * SM_SCALE
weights = torch.softmax(scores, dim=0)
output[b, h, :] = torch.mv(v.t(), weights).to(q.dtype)
return output
def _make_inputs(
batch_size: int,
nhead: int,
kv_seq_len: int,
*,
contiguous_indices: bool = True,
):
"""Build valid MLA decode inputs on the current CUDA device.
When contiguous_indices=False, tokens are randomly scattered in a 2x pool
to simulate real paged allocation.
"""
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAHelper
actual_nhead = AiterMLAHelper.get_actual_mla_num_heads(nhead)
total_kv_tokens = batch_size * kv_seq_len
pool_size = total_kv_tokens if contiguous_indices else total_kv_tokens * 2
q = torch.randn(
batch_size, actual_nhead, Q_HEAD_DIM, dtype=torch.bfloat16, device="cuda"
)
kv_buffer = torch.randn(
pool_size, 1, Q_HEAD_DIM, dtype=torch.bfloat16, device="cuda"
)
o = torch.zeros(
batch_size, actual_nhead, V_HEAD_DIM, dtype=torch.bfloat16, device="cuda"
)
qo_indptr = torch.arange(0, batch_size + 1, dtype=torch.int32, device="cuda")
kv_indptr = (
torch.arange(0, batch_size + 1, dtype=torch.int32, device="cuda") * kv_seq_len
)
kv_last_page_lens = torch.ones(batch_size, dtype=torch.int32, device="cuda")
if contiguous_indices:
kv_indices = torch.arange(0, total_kv_tokens, dtype=torch.int32, device="cuda")
else:
perm = torch.randperm(pool_size, device="cuda")[:total_kv_tokens]
kv_indices = perm.to(torch.int32)
return {
"q": q,
"kv_buffer": kv_buffer,
"o": o,
"qo_indptr": qo_indptr,
"kv_indptr": kv_indptr,
"kv_indices": kv_indices,
"kv_last_page_lens": kv_last_page_lens,
"actual_nhead": actual_nhead,
}
def _run_kernel(inputs: dict) -> torch.Tensor:
"""Run decode through the production rocm_aiter_ops path."""
from vllm._aiter_ops import rocm_aiter_ops
rocm_aiter_ops.mla_decode_fwd(
inputs["q"],
inputs["kv_buffer"],
inputs["o"],
SM_SCALE,
inputs["qo_indptr"],
1,
inputs["kv_indptr"],
inputs["kv_indices"],
inputs["kv_last_page_lens"],
)
return inputs["o"]
def _ref_output(inputs: dict) -> torch.Tensor:
"""Compute reference output from the same inputs."""
kv_flat = inputs["kv_buffer"].squeeze(1)
return _ref_mla_decode(
inputs["q"], kv_flat, inputs["kv_indptr"], inputs["kv_indices"]
)
@pytest.mark.parametrize("nhead", NUM_HEADS)
@pytest.mark.parametrize("batch_size", BATCH_SIZES)
@pytest.mark.parametrize("kv_seq_len", KV_SEQ_LENS)
@torch.inference_mode()
def test_mla_decode_accuracy(
nhead: int,
batch_size: int,
kv_seq_len: int,
) -> None:
"""BF16 decode accuracy vs PyTorch reference (contiguous indices)."""
_require_aiter()
set_random_seed(0)
inputs = _make_inputs(batch_size, nhead, kv_seq_len)
output = _run_kernel(inputs)
output_ref = _ref_output(inputs)
torch.testing.assert_close(output, output_ref, atol=ATOL, rtol=RTOL)
@pytest.mark.parametrize("nhead", NUM_HEADS)
@pytest.mark.parametrize("batch_size", [1, 4, 8])
@pytest.mark.parametrize("kv_seq_len", [16, 128])
@torch.inference_mode()
def test_mla_decode_noncontiguous_indices(
nhead: int,
batch_size: int,
kv_seq_len: int,
) -> None:
"""BF16 decode accuracy with shuffled kv_indices (paged allocation)."""
_require_aiter()
set_random_seed(0)
inputs = _make_inputs(batch_size, nhead, kv_seq_len, contiguous_indices=False)
output = _run_kernel(inputs)
output_ref = _ref_output(inputs)
torch.testing.assert_close(output, output_ref, atol=ATOL, rtol=RTOL)
@torch.inference_mode()
def test_mla_decode_determinism() -> None:
"""Repeated decode calls produce bitwise-identical results."""
_require_aiter()
set_random_seed(0)
inputs = _make_inputs(batch_size=4, nhead=128, kv_seq_len=16)
out_first = _run_kernel(inputs).clone()
for _ in range(3):
inputs["o"] = torch.zeros_like(inputs["o"])
out = _run_kernel(inputs)
torch.testing.assert_close(out, out_first, atol=0, rtol=0)
@torch.inference_mode()
def test_mla_decode_smoke() -> None:
"""Basic sanity: correct shape, dtype, finite, non-zero."""
_require_aiter()
set_random_seed(0)
inputs = _make_inputs(batch_size=4, nhead=128, kv_seq_len=64)
output = _run_kernel(inputs)
assert output.shape == (4, 128, V_HEAD_DIM)
assert output.dtype == torch.bfloat16
assert torch.isfinite(output).all()
assert not torch.all(output == 0)
@@ -1,213 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression test for AITER MLA persistent decode metadata dtypes.
For the gfx950 fp8/fp8 nhead=32 qlen=1 fold path, the split/reduce metadata
layout depends on the q/kv element size. The builder must forward dtype_q/dtype_kv
to ``get_mla_metadata_v1``; omitting them lays out the work for the wrong dtype
and corrupts decode output. The test pins the builder's metadata to a golden
recomputed at runtime with the explicit correct dtypes.
"""
import types
from unittest.mock import patch
import pytest
import torch
from vllm._aiter_ops import is_aiter_found
from vllm.platforms import current_platform
def _on_gfx950() -> bool:
if not (current_platform.is_rocm() and is_aiter_found()):
return False
from vllm.platforms.rocm import on_gfx950
return on_gfx950()
pytestmark = pytest.mark.skipif(
not _on_gfx950(),
reason="AITER MLA fp8 persistent decode metadata is gfx950-only",
)
# The fold path that the bug corrupted: fp8 query + fp8 KV-cache, 32 query
# heads, single-token decode, batch 128, context 8192, page_size 1.
NUM_QUERY_HEADS = 32
DECODE_QLEN = 1
BATCH_SIZE = 128
CONTEXT_LEN = 8192
PAGE_SIZE = 1
# On the fp8 KV-cache path the builder forwards the platform fp8 dtype (aiter
# dtypes.fp8) for both q and kv. Mirror it via current_platform.fp8_dtype()
# instead of hardcoding a literal (see #47276).
EXPECTED_Q_DTYPE = current_platform.fp8_dtype()
EXPECTED_KV_DTYPE = current_platform.fp8_dtype()
# The split/reduce content tensors filled by get_mla_metadata_v1. work_meta_data
# is excluded: it holds raw device pointers, never equal across allocations.
_CONTENT_METADATA_FIELDS = (
"work_indptr",
"work_info_set",
"reduce_indptr",
"reduce_final_map",
"reduce_partial_map",
)
# The builder's get_mla_metadata_v1 call passes 6 input args then 6 output
# buffers (see AiterMLAMetadataBuilder._build_decode). Output order -> field.
_NUM_INPUT_ARGS = 6
_OUTPUT_ARG_FIELDS = (
"work_meta_data",
"work_info_set",
"work_indptr",
"reduce_indptr",
"reduce_final_map",
"reduce_partial_map",
)
def _build_decode_metadata():
"""Build AITER MLA decode metadata for the fp8/fp8 nhead=32 fold path.
Returns ``(metadata, captured)`` where ``captured`` records the positional
args/kwargs the builder passed to ``get_mla_metadata_v1``, so the golden can
be recomputed from the identical inputs.
"""
from tests.v1.attention.utils import (
BatchSpec,
create_common_attn_metadata,
create_vllm_config,
)
from vllm.config.vllm import set_current_vllm_config
from vllm.model_executor.layers.attention.mla_attention import get_mla_dims
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.kv_cache_interface import MLAAttentionSpec
from vllm.v1.worker.workspace import init_workspace_manager
device = torch.device("cuda:0")
vllm_config = create_vllm_config(
model_name="deepseek-ai/DeepSeek-R1",
max_model_len=CONTEXT_LEN,
# One flat page per token (page_size=1); +buffer for the null block.
num_gpu_blocks=BATCH_SIZE * CONTEXT_LEN + 200,
block_size=PAGE_SIZE,
max_num_seqs=BATCH_SIZE,
max_num_batched_tokens=8192,
hf_config_override={"num_attention_heads": NUM_QUERY_HEADS},
)
vllm_config.cache_config.cache_dtype = "fp8"
spec = MLAAttentionSpec(
block_size=PAGE_SIZE,
num_kv_heads=1,
head_size=vllm_config.model_config.get_head_size(),
dtype=vllm_config.model_config.dtype,
cache_dtype_str="fp8",
)
builder_cls = AttentionBackendEnum.ROCM_AITER_MLA.get_class().get_builder_cls()
# The builder reads prefill_backend and the MLA latent dims off the layer,
# so the stub carries both; the dims come from the model config to stay in
# step with the checkpoint.
mla_dims = get_mla_dims(vllm_config.model_config)
layer_name = "placeholder"
vllm_config.compilation_config.static_forward_context[layer_name] = (
types.SimpleNamespace(
prefill_backend=torch.empty((1,)),
q_lora_rank=mla_dims.q_lora_rank,
kv_lora_rank=mla_dims.kv_lora_rank,
qk_nope_head_dim=mla_dims.qk_nope_head_dim,
qk_rope_head_dim=mla_dims.qk_rope_head_dim,
v_head_dim=mla_dims.v_head_dim,
)
)
init_workspace_manager(device)
batch_spec = BatchSpec(
seq_lens=[CONTEXT_LEN] * BATCH_SIZE,
query_lens=[DECODE_QLEN] * BATCH_SIZE,
)
captured: dict = {}
with set_current_vllm_config(vllm_config):
builder = builder_cls(spec, [layer_name], vllm_config, device)
common_attn_metadata = create_common_attn_metadata(
batch_spec, PAGE_SIZE, device, arange_block_indices=True
)
import aiter
real_get_mla_metadata_v1 = aiter.get_mla_metadata_v1
def spy(*args, **kwargs):
captured["args"] = args
captured["kwargs"] = dict(kwargs)
return real_get_mla_metadata_v1(*args, **kwargs)
with patch("aiter.get_mla_metadata_v1", spy):
metadata = builder.build(
common_prefix_len=0,
common_attn_metadata=common_attn_metadata,
)
return metadata, captured
def _compute_golden_metadata(captured: dict) -> dict[str, torch.Tensor]:
"""Recompute the persistent metadata with explicit fp8/bf16 dtypes.
Replays ``get_mla_metadata_v1`` on the builder's exact input tensors with
fresh output buffers and the explicitly-correct dtypes. This reference must
match the builder's output when the fix is in place.
"""
import aiter
args = captured["args"]
inputs = args[:_NUM_INPUT_ARGS]
# Fresh copies so the golden does not alias the builder's persistent buffers.
fresh_outputs = [arg.clone() for arg in args[_NUM_INPUT_ARGS:]]
golden_kwargs = dict(captured["kwargs"])
golden_kwargs["dtype_q"] = EXPECTED_Q_DTYPE
golden_kwargs["dtype_kv"] = EXPECTED_KV_DTYPE
aiter.get_mla_metadata_v1(*inputs, *fresh_outputs, **golden_kwargs)
return dict(zip(_OUTPUT_ARG_FIELDS, fresh_outputs))
def test_persistent_decode_metadata_matches_fp8_golden():
"""The builder's metadata must match the dtype-correct golden.
Regression guard: the fixed builder forwards fp8/bf16 dtypes so its
split/reduce metadata matches the golden recomputed with those explicit
dtypes. Dropping the dtypes (the original bug) produces a different layout
and fails this test.
"""
metadata, captured = _build_decode_metadata()
# qlen=1 must take the persistent-metadata path for this to be meaningful.
assert metadata.decode is not None
assert metadata.decode.has_persistent_metadata
assert metadata.work_meta_data is not None
golden = _compute_golden_metadata(captured)
mismatched = [
name
for name in _CONTENT_METADATA_FIELDS
if getattr(metadata, name).shape != golden[name].shape
or not torch.equal(getattr(metadata, name), golden[name])
]
assert not mismatched, (
"AITER MLA persistent decode metadata does not match the fp8/bf16 "
f"golden for fields {mismatched}; the builder must forward "
"dtype_q/dtype_kv to get_mla_metadata_v1."
)
@@ -1,246 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Accuracy test for the ROCm AITER FP8 MLA prefill path.
The FP8 MLA prefill path
(``AiterMLAImpl._mla_fp8_prefill_attn`` -> ``mla_prefill_ps_asm_fwd`` +
``mla_reduce_v1``) is auto-enabled on gfx950 (MI355) when AITER ships those
kernels, and it has no other test coverage. This test drives the *real*
metadata builder and impl methods (via ``object.__new__``, mirroring
``test_rocm_aiter_mla_head_padding.py::test_h12_aiter_mla_decode_matches_reference``)
so it exercises the actual persistent-scheduling metadata contract
(``get_ps_metadata_v1``) plus both kernels -- rather than re-implementing them --
and compares the output against a causal SDPA reference.
gfx950-only: ``mla_prefill_ps_asm_fwd`` has no gfx942 build, so this is skipped
on every other platform.
"""
import math
from types import SimpleNamespace
import pytest
import torch
import torch.nn.functional as F
from vllm.platforms import current_platform
def _fp8_prefill_available() -> bool:
if not (current_platform.is_rocm() and torch.cuda.is_available()):
return False
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
_fp8_mla_prefill_supported,
)
return _fp8_mla_prefill_supported()
pytestmark = pytest.mark.skipif(
not _fp8_prefill_available(),
reason="FP8 MLA prefill needs gfx950 (MI355) + an AITER build exporting "
"mla_prefill_ps_asm_fwd / mla_reduce_v1",
)
# DeepSeek-style MLA head dims after kv_b_proj decompression (q/k carry the
# nope+rope dims, v carries v_head_dim).
#
# The kernel requires 16-aligned heads, so _mla_fp8_prefill_attn replicate-pads
# q/k/v up to get_fp8_prefill_num_heads(num_heads) and slices the output back.
# Cover all four regimes:
# 8 -> padded to 16 (divisor, repeat_interleave), so the real heads sit at
# stride 2 in the padded output. Live case: Kimi-K2.x at TP8.
# 12 -> padded to 16 (non-divisor, tile+slice). Live case: a K3 rank at TP8.
# 16 -> no padding, output aliases the caller's buffer.
# 24 -> padded to 32. No current model and TP reaches this band (96 heads
# would need TP4, which does not fit in 288 GiB; 128-head models give
# 128/64/32/16/8), so this case exists to keep the general path honest
# for future architectures: it is the only one where padding above
# 16 actually happens, so it is what proves the replicate-pad/slice-back
# argument holds for a target other than 16.
HEAD_COUNTS = [8, 12, 16, 24]
QK_NOPE_HEAD_DIM = 128
QK_ROPE_HEAD_DIM = 64
QK_HEAD_DIM = QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM # 192
V_HEAD_DIM = 128
SCALE = 1.0 / math.sqrt(QK_HEAD_DIM)
# Dominated by fp8 (e4m3) rounding of q/k/v. Measured on MI355: worst-element
# abs error ~0.054, so atol=0.1 is ~2x that noise floor; rtol=0.05 bounds larger
# elements.
ATOL, RTOL = 1e-1, 5e-2
# The assembly kernel writes bf16 through a raw output pointer. Keep its tensors
# and workspace reservation on one dtype.
ATTN_OUT_DTYPE = torch.bfloat16
@pytest.fixture(autouse=True)
def _workspace_manager():
"""The FP8 prefill path draws per-call scratch from the global workspace
manager, which ``GPUModelRunner.__init__`` normally initializes. Set it up
for the test and tear it down so no global state leaks to other tests.
"""
from vllm.v1.worker.workspace import (
init_workspace_manager,
reset_workspace_manager,
)
init_workspace_manager(torch.device("cuda"))
yield
reset_workspace_manager()
def _make_impl(num_heads: int):
"""Minimal AiterMLAImpl exposing only what _mla_fp8_prefill_attn reads."""
from aiter import mla_prefill_ps_asm_fwd, mla_reduce_v1
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAImpl
impl = object.__new__(AiterMLAImpl)
impl.num_heads = num_heads
impl.v_head_dim = V_HEAD_DIM
impl.scale = SCALE
impl._mla_prefill_ps_asm_fwd = mla_prefill_ps_asm_fwd
impl._mla_reduce_v1 = mla_reduce_v1
return impl
def _build_prefill_metadata(
seq_lens: list[int],
device: torch.device,
num_heads: int,
attn_out_dtype: torch.dtype = ATTN_OUT_DTYPE,
):
"""Build metadata with the production builder and output dtype."""
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAMetadataBuilder
qo_indptr_cpu = torch.zeros(len(seq_lens) + 1, dtype=torch.int32)
qo_indptr_cpu[1:] = torch.tensor(seq_lens, dtype=torch.int32).cumsum(0)
qo_indptr = qo_indptr_cpu.to(device)
total_q = int(qo_indptr_cpu[-1].item())
max_q = max(seq_lens)
builder = object.__new__(AiterMLAMetadataBuilder)
builder.num_heads = num_heads
builder.mla_dims = SimpleNamespace(v_head_dim=V_HEAD_DIM)
builder._init_fp8_prefill_ps_buffers(
max_num_reqs=len(seq_lens),
max_prefill_qlen=max_q,
max_num_batched_tokens=total_q,
attn_out_dtype=attn_out_dtype,
device=device,
)
# SimpleNamespace stands in for the metadata dataclasses: the builder only
# reads prefill.{query_start_loc,max_query_len}, num_decodes, and
# common.query_start_loc_cpu, then populates metadata.fp8_prefill_*.
prefill = SimpleNamespace(query_start_loc=qo_indptr, max_query_len=max_q)
metadata = SimpleNamespace(prefill=prefill, num_decodes=0)
common = SimpleNamespace(query_start_loc_cpu=qo_indptr_cpu)
builder._build_fp8_prefill_ps_metadata(metadata, common)
return metadata, total_q, builder
def _reference(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor:
"""Causal SDPA reference on fp8-quantized inputs (matches kernel dtype)."""
fp8 = current_platform.fp8_dtype()
qf = q.to(fp8).float().transpose(0, 1) # [H, S, Dqk]
kf = k.to(fp8).float().transpose(0, 1)
vf = v.to(fp8).float().transpose(0, 1) # [H, S, Dv]
out = F.scaled_dot_product_attention(qf, kf, vf, is_causal=True, scale=SCALE)
return out.transpose(0, 1) # [S, H, Dv]
@pytest.mark.parametrize("num_heads", HEAD_COUNTS)
@pytest.mark.parametrize("seq_len", [128, 512])
@torch.inference_mode()
def test_fp8_prefill_matches_reference(seq_len: int, num_heads: int) -> None:
device = torch.device("cuda")
torch.manual_seed(0)
metadata, total_q, _ = _build_prefill_metadata([seq_len], device, num_heads)
# Lock the workspace, as GPUModelRunner does after warmup, so the forward
# cannot quietly grow it. The builder reserves scratch at num_head_k and
# the forward requests it at its own padded width; if a future change makes
# the forward's width exceed the builder's, that is an under-reservation
# which is invisible on an unlocked workspace (it just grows) and fatal in
# a real serve. Locking turns it into a failure here.
#
# Note this does not catch the reverse -- a builder wider than the forward
# merely over-reserves -- so it would not have flagged the pre-PR
# max(16, num_heads) sizing, which over-reserves 2.8x at 24 heads.
from vllm.v1.worker.workspace import lock_workspace
lock_workspace()
qkv_kwargs = dict(dtype=ATTN_OUT_DTYPE, device=device)
q = torch.randn(total_q, num_heads, QK_HEAD_DIM, **qkv_kwargs)
k = torch.randn(total_q, num_heads, QK_HEAD_DIM, **qkv_kwargs)
v = torch.randn(total_q, num_heads, V_HEAD_DIM, **qkv_kwargs)
out = torch.zeros(
total_q, num_heads * V_HEAD_DIM, dtype=ATTN_OUT_DTYPE, device=device
)
impl = _make_impl(num_heads)
impl._mla_fp8_prefill_attn(q, k, v, metadata, out)
out_ref = _reference(q, k, v)
assert torch.isfinite(out).all()
torch.testing.assert_close(
out.view(total_q, num_heads, V_HEAD_DIM).float(),
out_ref.float(),
atol=ATOL,
rtol=RTOL,
)
@pytest.mark.parametrize("num_heads", HEAD_COUNTS)
def test_fp8_prefill_metadata_width_matches_forward(num_heads: int) -> None:
"""The PS metadata must be built for the head count the kernel is handed.
``_init_fp8_prefill_ps_buffers``/``build()`` size the work and reduce maps
from ``num_head_k``, while ``_mla_fp8_prefill_attn`` pads q/k/v to its own
target. Divergence is not a numerical bug in either direction -- maps built
narrower than the kernel width still cover the real heads, since padding
only ever appends duplicate heads above them. It is a sizing bug: narrower
maps mean a lower head alignment, hence more partial tiles and a larger
reservation, while wider ones under-reserve the forward's scratch. Pin the
two to one helper rather than to independent expressions.
"""
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAHelper
width = AiterMLAHelper.get_fp8_prefill_num_heads(num_heads)
assert width % AiterMLAHelper._AITER_MIN_MLA_HEADS == 0
assert width >= num_heads
assert width - num_heads < AiterMLAHelper._AITER_MIN_MLA_HEADS
# Every count the enable gate accepts must have a usable padded width.
assert AiterMLAHelper.is_valid_num_heads(num_heads)
# The builder must actually size its maps at that width. Assert against the
# buffer it allocated rather than recomputing the width here, or the test
# passes no matter what _init_fp8_prefill_ps_buffers does.
#
# fp8_ps_reduce_partial_map is sized by the partial-tile count, which is
# gcd-driven and so is a sharp probe of the width used: a lower head
# alignment yields *more* tiles. At 24 heads, sizing at the unpadded count
# gives 4x the tiles (64 vs 16) and ~2.8x the scratch reservation (193.6
# vs 68.6 MiB at batch=1/qlen=512) -- not wrong numerically (the 24-wide
# maps still cover the real heads) but the reason to pin the widths
# together.
from aiter import get_ps_metadata_info_v1
from vllm.v1.attention.backends.mla.rocm_aiter_mla import _FP8_PREFILL_TILE_Q
def _tiles(nhk: int) -> int:
return get_ps_metadata_info_v1(
batch_size=1,
num_head_k=nhk,
max_qlen=512,
qlen_granularity=_FP8_PREFILL_TILE_Q,
)[5][0]
_, _, builder = _build_prefill_metadata([512], torch.device("cuda"), num_heads)
assert builder.fp8_ps_reduce_partial_map.numel() == _tiles(width)
@@ -1,252 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for ROCm AITER MLA FP8 support detection."""
import sys
import types
from typing import Any
from unittest.mock import patch
import pytest
from vllm.platforms import current_platform
_SKIP_UNSUPPORTED_AITER_HARDWARE = True
if current_platform.is_rocm():
from vllm.platforms.rocm import get_cdna_version
_SKIP_UNSUPPORTED_AITER_HARDWARE = get_cdna_version() <= 2
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(), reason="ROCm-specific tests"
)
@pytest.fixture(autouse=True)
def reset_aiter_mla_support_cache(monkeypatch: pytest.MonkeyPatch) -> None:
import vllm._aiter_ops as aiter_ops
monkeypatch.setattr(aiter_ops, "_AITER_MLA_SUPPORTS_FP8", None)
def _install_fake_aiter_modules(
monkeypatch: pytest.MonkeyPatch,
*,
supports_fp8: bool,
supports_causal: bool = False,
) -> None:
aiter_mod: Any = types.ModuleType("aiter")
mla_mod: Any = types.ModuleType("aiter.mla")
if supports_fp8:
def mla_decode_fwd_with_fp8(
q,
kv_buffer,
kv_indptr,
kv_indices,
o,
sm_scale,
q_scale=None,
kv_scale=None,
):
return None
mla_decode_fwd: Any = mla_decode_fwd_with_fp8
else:
def mla_decode_fwd_without_fp8(
q,
kv_buffer,
kv_indptr,
kv_indices,
o,
sm_scale,
):
return None
mla_decode_fwd = mla_decode_fwd_without_fp8
if supports_causal:
inner = mla_decode_fwd
def mla_decode_fwd_with_causal(*args, causal=True, **kwargs):
return inner(*args, **kwargs)
mla_decode_fwd = mla_decode_fwd_with_causal
mla_mod.mla_decode_fwd = mla_decode_fwd
aiter_mod.mla = mla_mod
monkeypatch.setitem(sys.modules, "aiter", aiter_mod)
monkeypatch.setitem(sys.modules, "aiter.mla", mla_mod)
def test_aiter_mla_fp8_support_detects_fp8_signature(monkeypatch):
"""The support check should detect q_scale and kv_scale parameters."""
from vllm._aiter_ops import _check_aiter_mla_fp8_support
_install_fake_aiter_modules(monkeypatch, supports_fp8=True)
assert _check_aiter_mla_fp8_support() is True
def test_aiter_mla_fp8_support_rejects_missing_fp8_signature(monkeypatch):
"""The support check should return False when FP8 params are absent."""
from vllm._aiter_ops import _check_aiter_mla_fp8_support
_install_fake_aiter_modules(monkeypatch, supports_fp8=False)
assert _check_aiter_mla_fp8_support() is False
@pytest.mark.skipif(
_SKIP_UNSUPPORTED_AITER_HARDWARE,
reason="Installed AITER MLA FP8 check requires CDNA 3 or newer",
)
def test_installed_aiter_mla_supports_fp8():
"""Supported ROCm CI must provide AITER with MLA FP8 scaling."""
from vllm._aiter_ops import (
_check_aiter_mla_fp8_support,
is_aiter_found_and_supported,
)
assert is_aiter_found_and_supported(), (
"AITER must be installed on supported ROCm hardware"
)
assert _check_aiter_mla_fp8_support() is True
@pytest.mark.parametrize(
"error_type",
[ImportError, ModuleNotFoundError, AttributeError, ValueError, TypeError],
)
def test_aiter_mla_fp8_support_handles_signature_errors(monkeypatch, error_type):
"""The support check should fail closed on import or signature problems."""
import vllm._aiter_ops as aiter_ops
from vllm._aiter_ops import _check_aiter_mla_fp8_support
_install_fake_aiter_modules(monkeypatch, supports_fp8=True)
with patch("inspect.signature", side_effect=error_type("boom")):
assert _check_aiter_mla_fp8_support() is False
assert aiter_ops._AITER_MLA_SUPPORTS_FP8 is False
def test_aiter_mla_fp8_support_result_is_cached(monkeypatch):
"""The support check should reuse the cached result on later calls."""
import inspect
from vllm._aiter_ops import _check_aiter_mla_fp8_support
_install_fake_aiter_modules(monkeypatch, supports_fp8=True)
with patch("inspect.signature", wraps=inspect.signature) as signature_mock:
assert _check_aiter_mla_fp8_support() is True
assert _check_aiter_mla_fp8_support() is True
assert signature_mock.call_count == 1
def test_missing_causal_arg_fails_closed(monkeypatch):
"""Old aiter wheels have no causal=; do not report that as a valid draft path."""
from vllm._aiter_ops import rocm_aiter_ops
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLABackend
_install_fake_aiter_modules(monkeypatch, supports_fp8=True, supports_causal=False)
with pytest.raises(RuntimeError, match="causal-only"):
rocm_aiter_ops.mla_decode_supports_non_causal()
with pytest.raises(RuntimeError, match="causal-only"):
AiterMLABackend.supports_non_causal()
@pytest.mark.parametrize("has_kernels", [False, True])
def test_supports_non_causal_requires_arch_kernels(monkeypatch, has_kernels):
"""The Python argument alone does not guarantee an arch has the kernel."""
from vllm._aiter_ops import rocm_aiter_ops
from vllm.v1.attention.backends.mla import rocm_aiter_mla
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLABackend
_install_fake_aiter_modules(monkeypatch, supports_fp8=True, supports_causal=True)
monkeypatch.setattr(
rocm_aiter_mla,
"_aiter_mla_non_causal_asm_kernels",
lambda: has_kernels,
)
assert rocm_aiter_ops.mla_decode_supports_non_causal() is True
assert AiterMLABackend.supports_non_causal() is has_kernels
@pytest.mark.parametrize("causal", [True, False])
def test_decode_fwd_forwards_causal_to_aiter(monkeypatch, causal):
"""The decode op must pass the block mask through; omitting it is causal-only."""
seen: list = []
def mla_decode_fwd(*args, causal=None, **kwargs):
seen.append(causal)
mla_mod: Any = types.ModuleType("aiter.mla")
mla_mod.mla_decode_fwd = mla_decode_fwd
aiter_mod: Any = types.ModuleType("aiter")
aiter_mod.mla = mla_mod
monkeypatch.setitem(sys.modules, "aiter", aiter_mod)
monkeypatch.setitem(sys.modules, "aiter.mla", mla_mod)
import torch
import vllm._aiter_ops as aiter_ops
aiter_ops._rocm_aiter_mla_decode_fwd_impl(
torch.zeros(2, 1, 8),
torch.zeros(2, 1, 1, 8),
torch.zeros(2, 1, 8),
torch.zeros(3, dtype=torch.int32),
1,
causal=causal,
)
assert seen == [causal]
def _aiter_mla_validate(
monkeypatch,
*,
dtype,
kv_cache_dtype,
use_non_causal,
):
from vllm.platforms.interface import DeviceCapability
from vllm.v1.attention.backend import AttentionType
from vllm.v1.attention.backends.mla import rocm_aiter_mla
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLABackend
_install_fake_aiter_modules(monkeypatch, supports_fp8=True, supports_causal=True)
monkeypatch.setattr(
rocm_aiter_mla, "_aiter_mla_non_causal_asm_kernels", lambda: True
)
return AiterMLABackend.validate_configuration(
head_size=576,
dtype=dtype,
kv_cache_dtype=kv_cache_dtype,
block_size=1,
use_mla=True,
has_sink=False,
use_sparse=False,
use_mm_prefix=False,
use_per_head_quant_scales=False,
device_capability=DeviceCapability(9, 5),
attn_type=AttentionType.DECODER,
use_non_causal=use_non_causal,
)
def test_non_causal_fp16_auto_cache_is_rejected(monkeypatch):
"""Pinned AITER aborts fp16 Q; auto-select must fall through to Triton."""
import torch
reasons = _aiter_mla_validate(
monkeypatch,
dtype=torch.float16,
kv_cache_dtype="auto",
use_non_causal=True,
)
assert any("non-causal fp16" in reason for reason in reasons)
@@ -1,357 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Head padding and kernel selection for the ROCm AITER MLA backend.
The asm persistent decode requires a 16-aligned head count, so unaligned
counts through 128 are tile-padded to the next multiple of 16 and sliced back
off the output. Small divisor counts (1/2/4/8) preserve their existing
repeat-interleave path and may keep the Gluon kernel on gfx950.
"""
import math
import types
import pytest
import torch
import torch.nn.functional as F
from vllm._aiter_ops import is_aiter_found
from vllm.platforms import current_platform
from vllm.v1.attention.backends.mla import rocm_aiter_mla
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
AiterMLADecodeMetadata,
AiterMLAHelper,
AiterMLAImpl,
AiterMLAMetadata,
)
NUM_HEADS = 12
KV_LORA_RANK = 512
QK_ROPE_HEAD_DIM = 64
QK_HEAD_DIM = KV_LORA_RANK + QK_ROPE_HEAD_DIM
CONTEXT_LEN = 4096
SCALE = 1.0 / math.sqrt(QK_HEAD_DIM)
# Small non-divisor counts use tile-and-slice; divisors of 16 keep
# repeat_interleave. Both pad to exactly 16 and round-trip.
NON_DIVISOR_HEADS = [3, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15]
DIVISOR_HEADS = [1, 2, 4, 8]
@pytest.fixture(autouse=True)
def _disable_native_h24(monkeypatch):
"""Exercise the padding fallback unless a test explicitly enables H24."""
monkeypatch.setattr(
rocm_aiter_mla, "_aiter_mla_native_h24_supported", lambda: False
)
def _rocm_aiter_available() -> bool:
return current_platform.is_rocm() and is_aiter_found() and torch.cuda.is_available()
def _on_gfx950() -> bool:
if not (current_platform.is_rocm() and is_aiter_found()):
return False
from vllm.platforms.rocm import on_gfx950
return on_gfx950()
def _expected_tile_pad(q: torch.Tensor, num_heads: int, m: int = 16) -> torch.Tensor:
# get_mla_padded_q tiles the heads and slices to m, i.e. head i of the
# padded tensor is head (i % num_heads) of the input.
idx = [i % num_heads for i in range(m)]
return q[:, idx, :]
def _make_h12_decode_metadata(device: torch.device) -> AiterMLAMetadata:
page_indices = torch.arange(CONTEXT_LEN, dtype=torch.int32, device=device)
decode = AiterMLADecodeMetadata(
block_table=page_indices.view(1, -1),
seq_lens=torch.tensor([CONTEXT_LEN], dtype=torch.int32, device=device),
dcp_tot_seq_lens=None,
paged_kv_indptr=torch.tensor(
[0, CONTEXT_LEN], dtype=torch.int32, device=device
),
paged_kv_indices=page_indices,
paged_kv_last_page_len=torch.ones(1, dtype=torch.int32, device=device),
qo_indptr=torch.tensor([0, 1], dtype=torch.int32, device=device),
attn_out_dtype=torch.bfloat16,
max_qo_len=1,
)
return AiterMLAMetadata(
num_reqs=1,
max_query_len=1,
max_seq_len=CONTEXT_LEN,
num_actual_tokens=1,
query_start_loc=decode.qo_indptr,
slot_mapping=torch.tensor([CONTEXT_LEN - 1], dtype=torch.int64, device=device),
num_decodes=1,
num_decode_tokens=1,
num_prefills=0,
decode=decode,
)
def test_h12_query_is_tile_padded_to_h16():
q = torch.arange(2 * 12 * 4, dtype=torch.bfloat16).view(2, 12, 4)
padded_q = AiterMLAHelper.get_mla_padded_q(12, q)
assert padded_q.shape == (2, 16, 4)
assert padded_q.is_contiguous()
# The real heads are untouched...
torch.testing.assert_close(padded_q[:, :12], q)
# ...and the 4 padding heads are the tiled wrap-around (heads 0..3), not
# zeros: MLA attention is per-head independent so duplicate query heads are
# harmless and get sliced back off the output.
torch.testing.assert_close(padded_q[:, 12:], q[:, :4])
def test_h6_tp16_query_is_padded_to_h16():
# TP16 puts 6 heads/rank. The old append-only padding produced cat(6, 6) =
# 12 heads and broke the asm kernel; tile-and-slice must reach exactly 16.
q = torch.arange(2 * 6 * 4, dtype=torch.bfloat16).view(2, 6, 4)
padded_q = AiterMLAHelper.get_mla_padded_q(6, q)
assert padded_q.shape == (2, 16, 4)
assert padded_q.is_contiguous()
torch.testing.assert_close(padded_q, _expected_tile_pad(q, 6))
def test_h12_output_discards_padding_heads():
o = torch.arange(2 * 16 * 4, dtype=torch.bfloat16).view(2, 16, 4)
unpadded_o = AiterMLAHelper.get_mla_unpadded_o(12, o)
assert unpadded_o.shape == (2, 12, 4)
torch.testing.assert_close(unpadded_o, o[:, :12])
def test_h24_query_is_tile_padded_to_h32():
q = torch.arange(2 * 24 * 4, dtype=torch.float32).view(2, 24, 4)
padded_q = AiterMLAHelper.get_mla_padded_q(24, q)
assert padded_q.shape == (2, 32, 4)
assert padded_q.is_contiguous()
torch.testing.assert_close(padded_q[:, :24], q)
torch.testing.assert_close(padded_q[:, 24:], q[:, :8])
def test_h24_output_discards_h32_padding_heads():
o = torch.arange(2 * 32 * 4, dtype=torch.float32).view(2, 32, 4)
unpadded_o = AiterMLAHelper.get_mla_unpadded_o(24, o)
assert unpadded_o.shape == (2, 24, 4)
torch.testing.assert_close(unpadded_o, o[:, :24])
def test_h24_reducer_without_metadata_still_pads_to_h32(monkeypatch):
monkeypatch.setattr(
rocm_aiter_mla, "_aiter_mla_native_h24_reducer_supported", lambda: True
)
monkeypatch.setattr(
rocm_aiter_mla, "_aiter_mla_native_h24_metadata_supported", lambda: False
)
monkeypatch.setattr(
rocm_aiter_mla,
"_aiter_mla_native_h24_supported",
lambda: (
rocm_aiter_mla._aiter_mla_native_h24_reducer_supported()
and rocm_aiter_mla._aiter_mla_native_h24_metadata_supported()
),
)
q = torch.arange(2 * 24 * 4, dtype=torch.float32).view(2, 24, 4)
assert AiterMLAHelper.get_actual_mla_num_heads(24) == 32
padded_q = AiterMLAHelper.get_mla_padded_q(24, q)
assert padded_q.shape == (2, 32, 4)
torch.testing.assert_close(padded_q[:, :24], q)
torch.testing.assert_close(padded_q[:, 24:], q[:, :8])
def test_native_h24_requires_reducer_and_metadata(monkeypatch):
monkeypatch.setattr(
rocm_aiter_mla, "_aiter_mla_native_h24_reducer_supported", lambda: True
)
monkeypatch.setattr(
rocm_aiter_mla, "_aiter_mla_native_h24_metadata_supported", lambda: True
)
monkeypatch.setattr(
rocm_aiter_mla,
"_aiter_mla_native_h24_supported",
lambda: (
rocm_aiter_mla._aiter_mla_native_h24_reducer_supported()
and rocm_aiter_mla._aiter_mla_native_h24_metadata_supported()
),
)
q = torch.arange(2 * 24 * 4, dtype=torch.float32).view(2, 24, 4)
assert AiterMLAHelper.get_actual_mla_num_heads(24) == 24
assert AiterMLAHelper.get_mla_padded_q(24, q) is q
assert AiterMLAHelper.get_mla_unpadded_o(24, q) is q
def test_existing_divisor_head_mapping_is_unchanged():
q = torch.arange(2 * 8 * 4, dtype=torch.bfloat16).view(2, 8, 4)
padded_q = AiterMLAHelper.get_mla_padded_q(8, q)
unpadded_o = AiterMLAHelper.get_mla_unpadded_o(8, padded_q)
# Divisor counts still use repeat_interleave / strided unpad, unchanged.
torch.testing.assert_close(padded_q, q.repeat_interleave(2, dim=1))
torch.testing.assert_close(unpadded_o, q)
@pytest.mark.parametrize("num_heads", [17, 24, 31])
def test_unaligned_head_counts_round_trip_through_h32(num_heads: int):
q = torch.arange(2 * num_heads * 4, dtype=torch.float32).view(2, num_heads, 4)
padded_q = AiterMLAHelper.get_mla_padded_q(num_heads, q)
unpadded_o = AiterMLAHelper.get_mla_unpadded_o(num_heads, padded_q)
assert padded_q.shape == (2, 32, 4)
assert padded_q.is_contiguous()
torch.testing.assert_close(unpadded_o, q)
@pytest.mark.parametrize("num_heads", NON_DIVISOR_HEADS + DIVISOR_HEADS)
def test_all_small_head_counts_pad_to_16_and_round_trip(num_heads: int):
q = torch.arange(2 * num_heads * 4, dtype=torch.float32).view(2, num_heads, 4)
padded_q = AiterMLAHelper.get_mla_padded_q(num_heads, q)
unpadded_o = AiterMLAHelper.get_mla_unpadded_o(num_heads, padded_q)
assert padded_q.shape == (2, 16, 4)
assert padded_q.is_contiguous()
# The real heads survive the pad -> unpad round trip exactly.
torch.testing.assert_close(unpadded_o, q)
def test_aligned_h32_is_zero_copy():
q = torch.arange(2 * 32 * 4, dtype=torch.float32).view(2, 32, 4)
assert AiterMLAHelper.get_mla_padded_q(32, q) is q
assert AiterMLAHelper.get_mla_unpadded_o(32, q) is q
def test_is_valid_num_heads():
for n in range(1, 129):
assert AiterMLAHelper.is_valid_num_heads(n)
assert AiterMLAHelper.is_valid_num_heads(24)
assert AiterMLAHelper.is_valid_num_heads(127)
# Aligned counts remain valid above the range where padding is supported.
assert AiterMLAHelper.is_valid_num_heads(144)
assert not AiterMLAHelper.is_valid_num_heads(0)
assert not AiterMLAHelper.is_valid_num_heads(129)
def test_nondivisor_and_multitoken_never_use_gluon():
# Non-divisor decode always takes the asm path (12 heads/rank at TP8).
assert not AiterMLAHelper.use_gluon_decode(12, 1, "auto")
assert not AiterMLAHelper.use_gluon_decode(6, 1, "auto")
# >=16 heads never use Gluon, including unaligned counts padded for asm.
assert not AiterMLAHelper.use_gluon_decode(16, 1, "auto")
assert not AiterMLAHelper.use_gluon_decode(24, 1, "auto")
# Multi-token (verify / qlen>1) is never the single-token Gluon decode.
assert not AiterMLAHelper.use_gluon_decode(8, 4, "auto")
assert not AiterMLAHelper.use_gluon_decode(12, 4, "auto")
def test_divisor_gluon_selection_follows_arch():
# Divisor head counts keep Gluon only where the kernel exists (gfx950).
# On gfx942 / non-ROCm they must route to the asm persistent decode.
on_gfx950 = _on_gfx950()
assert AiterMLAHelper.use_gluon_decode(8, 1, "auto") is on_gfx950
assert AiterMLAHelper.use_gluon_decode(4, 1, "auto") is on_gfx950
assert AiterMLAHelper.use_gluon_decode(1, 1, "auto") is on_gfx950
def test_asm_padding_env_default_is_auto(monkeypatch):
monkeypatch.delenv("VLLM_ROCM_AITER_MLA_ASM_PADDING", raising=False)
import vllm.envs as envs
assert envs.VLLM_ROCM_AITER_MLA_ASM_PADDING == "auto"
def test_asm_padding_env_force_asm_disables_gluon(monkeypatch):
monkeypatch.setenv("VLLM_ROCM_AITER_MLA_ASM_PADDING", "asm")
# Forcing the asm path: no small-head count uses Gluon on any arch.
for num_heads in (1, 2, 4, 8, 12):
assert not AiterMLAHelper.use_gluon_decode(num_heads, 1, "auto")
def test_asm_padding_env_force_gluon_follows_arch(monkeypatch):
monkeypatch.setenv("VLLM_ROCM_AITER_MLA_ASM_PADDING", "gluon")
on_gfx950 = _on_gfx950()
# Forcing Gluon: any 1..15 single-token decode uses it where a build exists
# (gfx950), including non-divisor counts like 12; gfx942/non-ROCm still
# falls back to the asm path.
for num_heads in (1, 2, 4, 8, 12):
assert AiterMLAHelper.use_gluon_decode(num_heads, 1, "auto") is on_gfx950
def test_asm_padding_env_auto_matches_arch_gate(monkeypatch):
monkeypatch.setenv("VLLM_ROCM_AITER_MLA_ASM_PADDING", "auto")
on_gfx950 = _on_gfx950()
# auto: divisor counts keep Gluon on gfx950, non-divisor counts take asm.
assert AiterMLAHelper.use_gluon_decode(8, 1, "auto") is on_gfx950
assert not AiterMLAHelper.use_gluon_decode(12, 1, "auto")
@pytest.mark.skipif(
not _rocm_aiter_available(),
reason="12-head AITER MLA asm persistent decode needs ROCm + AITER",
)
@torch.inference_mode()
def test_h12_aiter_mla_decode_matches_reference():
"""The 12-head AITER decode output must match an attention reference.
12 is a non-divisor of 16 so use_gluon_decode is False on every arch; the
query is tile-padded to 16 heads and served by the asm persistent decode
(the CDNA3 path this PR enables, and unchanged on CDNA4).
"""
device = torch.device("cuda:0")
torch.manual_seed(42)
q = torch.randn(1, NUM_HEADS, QK_HEAD_DIM, dtype=torch.bfloat16, device=device)
kv_cache = torch.randn(
CONTEXT_LEN, 1, QK_HEAD_DIM, dtype=torch.bfloat16, device=device
)
impl = object.__new__(AiterMLAImpl)
impl.num_heads = NUM_HEADS
impl.dcp_world_size = 1
impl.kv_lora_rank = KV_LORA_RANK
impl.qk_rope_head_dim = QK_ROPE_HEAD_DIM
impl.scale = SCALE
impl.kv_cache_dtype = "auto"
one = torch.ones(1, dtype=torch.float32, device=device)
layer = types.SimpleNamespace(_q_scale=one, _k_scale=one)
metadata = _make_h12_decode_metadata(device)
out, _ = impl.forward_mqa(q, kv_cache, metadata, layer)
assert out.shape[1] == NUM_HEADS # padding heads sliced back off
key = kv_cache[:, 0].float().unsqueeze(0)
value = key[..., :KV_LORA_RANK]
out_ref = F.scaled_dot_product_attention(
q[0].float().unsqueeze(1),
key,
value,
scale=SCALE,
enable_gqa=True,
).squeeze(1)
assert torch.isfinite(out).all()
torch.testing.assert_close(
out.float(),
out_ref.unsqueeze(0),
atol=1e-2,
rtol=1e-2,
)
@@ -1,523 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness tests for ROCm AITER sparse MLA attention sinks."""
from types import SimpleNamespace
import pytest
import torch
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(), reason="ROCm-specific tests"
)
Q_HEAD_DIM = 576
V_HEAD_DIM = 512
SM_SCALE = Q_HEAD_DIM**-0.5
def _gamma_fp32(operations: int) -> float:
u = torch.finfo(torch.float32).eps / 2
return operations * u / (1 - operations * u)
def _sink_reference(q, keys, sinks, scale):
# Reference the stored operands, so input quantization is not kernel error.
q, keys, sinks = q.double(), keys.double(), sinks.double()
scores = q @ keys.T * scale
logits = torch.cat((scores, sinks[:, None]), dim=-1)
probabilities = logits.softmax(dim=-1)[:, :-1]
values = keys[:, :V_HEAD_DIM]
expected = probabilities @ values
value_scale = probabilities @ values.abs()
# FP32 dot accumulation and conversion/multiplication of the scale.
score_error = _gamma_fp32(q.shape[-1] + 2) * (q.abs() @ keys.abs().T) * abs(scale)
score_error = score_error.amax(dim=-1) if keys.shape[0] else torch.zeros_like(sinks)
return expected, logits.logsumexp(dim=-1), value_scale, score_error
def _assert_sink_output_close(
actual, expected, value_scale, score_error, probability_dtype, native
):
assert actual.shape == expected.shape
# Budget the identifiable rounding stages against the absolute value
# contributions: output-relative ULPs become unbounded when values cancel.
# Rounding P before P@V costs u(P) * sum(P*abs(V)).
# AITER rounds output before and after sink scaling; Triton rounds once.
u_probability = torch.finfo(probability_dtype).eps / 2
u_output = torch.finfo(actual.dtype).eps / 2
output_rounds = 2 if native else 1
amplification = (1 + u_output) ** output_rounds
relative_error = (1 + u_probability) * torch.exp(2 * score_error) - 1
allowance = (
relative_error.unsqueeze(-1) * value_scale * amplification
+ (amplification - 1) * expected.abs()
+ output_rounds * torch.finfo(actual.dtype).tiny * u_output
)
actual = actual.to(device=expected.device, dtype=torch.float64)
assert torch.isfinite(actual).all()
assert torch.count_nonzero(actual[value_scale == 0]) == 0
error = (actual - expected).abs()
max_ratio = (error / allowance.clamp_min(1e-300)).max().item()
assert max_ratio <= 1, f"exceeded rounding budget by {max_ratio:.3f}x"
def _require_aiter() -> None:
from vllm._aiter_ops import is_aiter_found_and_supported
from vllm.platforms.rocm import get_cdna_version
if get_cdna_version() not in (3, 4):
pytest.skip("AITER MLA requires CDNA 3 or 4")
if not is_aiter_found_and_supported():
pytest.skip("aiter is required on supported ROCm hardware for this test")
@pytest.mark.parametrize(
("real_heads", "cache_kind"),
[
(1, "bf16"),
(4, "bf16"),
(8, "bf16"),
(8, "fp8"),
(32, "fp8"),
(64, "fp8"),
(12, "bf16"),
(16, "bf16"),
(24, "bf16"),
(32, "bf16"),
(40, "bf16"),
(48, "bf16"),
(64, "bf16"),
(80, "bf16"),
],
)
@torch.inference_mode()
def test_sparse_mla_sink_matches_ragged_reference(
real_heads: int, cache_kind: str
) -> None:
"""Exercise ragged sink decode, head padding, and FP8 scale forwarding."""
_require_aiter()
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAHelper
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
ROCMAiterMLASparseImpl,
)
set_random_seed(real_heads * 17 + (cache_kind == "fp8"))
device = torch.device("cuda")
seq_lens = [0, 1, 5, 23, 17]
batch_size = len(seq_lens)
q_source = torch.randn(batch_size, real_heads, Q_HEAD_DIM, device=device) * 0.2
pool_size = sum(seq_lens) + 52
kv_source = torch.randn(pool_size, 1, Q_HEAD_DIM, device=device) * 0.2
if cache_kind == "fp8":
fp8_dtype = current_platform.fp8_dtype()
q_scale = torch.tensor(0.5, dtype=torch.float32, device=device)
kv_scale = torch.tensor(0.25, dtype=torch.float32, device=device)
q_real = (q_source / q_scale).to(fp8_dtype)
kv = (kv_source / kv_scale).to(fp8_dtype)
q_ref = q_real.float() * q_scale
kv_ref = kv.float() * kv_scale
else:
q_scale = kv_scale = None
q_real = q_source.to(torch.bfloat16)
kv = kv_source.to(torch.bfloat16)
q_ref = q_real.float()
kv_ref = kv.float()
q = AiterMLAHelper.get_mla_padded_q(real_heads, q_real)
indices = torch.randperm(pool_size, device=device)[: sum(seq_lens)].to(torch.int32)
kv_indptr = torch.tensor(
[0] + [sum(seq_lens[:i]) for i in range(1, len(seq_lens) + 1)],
dtype=torch.int32,
device=device,
)
# Non-None garbage proves the sink path cannot accidentally select the
# gfx942 persistent kernel, which has no return-LSE code object.
metadata = SimpleNamespace(
attn_out_dtype=torch.bfloat16,
qo_indptr=torch.arange(batch_size + 1, dtype=torch.int32, device=device),
paged_kv_indptr=kv_indptr,
paged_kv_indices=indices,
paged_kv_last_page_len=torch.ones(batch_size, dtype=torch.int32, device=device),
work_meta_data=torch.tensor([123], dtype=torch.int32),
work_indptr=None,
work_info_set=None,
reduce_indptr=None,
reduce_final_map=None,
reduce_partial_map=None,
num_prefills=0,
num_decodes=batch_size,
num_decode_tokens=batch_size,
max_query_len=1,
max_seq_len=max(seq_lens),
topk_tokens=max(seq_lens),
)
sinks = torch.linspace(-2.0, 6.0, real_heads, device=device)
impl = object.__new__(ROCMAiterMLASparseImpl)
impl.num_heads = real_heads
impl.kv_lora_rank = V_HEAD_DIM
impl.kv_cache_dtype = cache_kind
impl.scale = SM_SCALE
impl.sinks = sinks
output, lse = impl._forward_mla(
SimpleNamespace(_q_scale=q_scale, _k_scale=kv_scale),
q,
kv,
metadata,
)
kv_flat = kv_ref[:, 0]
references = []
start = 0
for batch_idx, seq_len in enumerate(seq_lens):
rows = kv_flat[indices[start : start + seq_len].long()]
references.append(_sink_reference(q_ref[batch_idx], rows, sinks, SM_SCALE))
start += seq_len
expected, expected_lse, value_scale, score_error = (
torch.stack(values) for values in zip(*references)
)
assert output.dtype == torch.bfloat16
if lse is not None:
assert lse.dtype == torch.float32
assert lse.shape == expected_lse.shape
lse_error = (lse.double() - expected_lse).abs()
rounded_lse = expected_lse.float()
positive_inf = torch.full_like(rounded_lse, float("inf"))
ulp = torch.maximum(
torch.nextafter(rounded_lse, positive_inf) - rounded_lse,
rounded_lse - torch.nextafter(rounded_lse, -positive_inf),
).double()
lse_allowance = score_error + 2 * ulp
assert torch.all(lse_error <= lse_allowance)
torch.testing.assert_close(lse[0].double(), expected_lse[0], atol=0, rtol=0)
else:
from vllm.platforms.rocm import on_gfx942
assert cache_kind == "bf16" and real_heads in (40, 48, 64) and on_gfx942()
_assert_sink_output_close(
output,
expected,
value_scale,
score_error,
q_real.dtype,
native=lse is not None,
)
@pytest.mark.parametrize(
("q_dtype", "kv_dtype"),
[
(torch.float16, torch.bfloat16),
(torch.bfloat16, torch.float16),
],
)
def test_sparse_mla_sink_rejects_unsupported_aiter_dtypes(
q_dtype: torch.dtype, kv_dtype: torch.dtype
) -> None:
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
ROCMAiterMLASparseImpl,
)
impl = object.__new__(ROCMAiterMLASparseImpl)
impl.num_heads = 16
impl.kv_lora_rank = V_HEAD_DIM
impl.kv_cache_dtype = "auto"
impl.scale = SM_SCALE
impl.sinks = torch.zeros(16, dtype=torch.float32)
metadata = SimpleNamespace(
attn_out_dtype=torch.bfloat16,
num_prefills=0,
num_decodes=1,
num_decode_tokens=1,
max_query_len=1,
)
with pytest.raises(ValueError, match="both use BF16 or both use FP8"):
impl._forward_mla(
SimpleNamespace(_q_scale=None, _k_scale=None),
torch.empty(1, 16, Q_HEAD_DIM, dtype=q_dtype),
torch.empty(1, 1, Q_HEAD_DIM, dtype=kv_dtype),
metadata,
)
def _make_noncontiguous_sink() -> torch.Tensor:
return torch.empty(8, dtype=torch.float32)[::2]
@pytest.mark.parametrize(
("sinks", "match"),
[
(torch.empty(4, dtype=torch.bfloat16), "must be float32"),
(torch.empty(2, 2, dtype=torch.float32), "must have shape"),
(_make_noncontiguous_sink(), "must be contiguous"),
],
)
def test_sparse_mla_sink_validation(sinks: torch.Tensor, match: str) -> None:
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
ROCMAiterMLASparseImpl,
)
with pytest.raises(ValueError, match=match):
ROCMAiterMLASparseImpl(
num_heads=4,
head_size=Q_HEAD_DIM,
scale=SM_SCALE,
num_kv_heads=1,
alibi_slopes=None,
sliding_window=None,
kv_cache_dtype="bfloat16",
logits_soft_cap=None,
attn_type="decoder",
kv_sharing_target_layer_name=None,
sinks=sinks,
kv_lora_rank=V_HEAD_DIM,
)
def test_sparse_mla_backend_reports_sink_support_for_current_hardware() -> None:
from vllm.platforms.rocm import get_cdna_version
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
ROCMAiterMLASparseBackend,
)
assert ROCMAiterMLASparseBackend.supports_sink() == (get_cdna_version() in (3, 4))
def test_sparse_mla_backend_rejects_dcp() -> None:
from vllm.platforms.rocm import RocmPlatform
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.attention.selector import AttentionSelectorConfig
selector_config = AttentionSelectorConfig(
head_size=Q_HEAD_DIM,
dtype=torch.bfloat16,
kv_cache_dtype="bfloat16",
block_size=16,
use_mla=True,
has_sink=True,
use_sparse=True,
use_mm_prefix=False,
use_per_head_quant_scales=False,
attn_type="decoder",
use_dcp=True,
)
with pytest.raises(ValueError, match="DCP not supported"):
RocmPlatform.get_attn_backend_cls(
selected_backend=AttentionBackendEnum.ROCM_AITER_MLA_SPARSE,
attn_selector_config=selector_config,
)
@pytest.mark.parametrize("num_heads,block_size", [(8, 16), (20, 32)])
@pytest.mark.parametrize("interleaved_pages", [False, True])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@torch.inference_mode()
def test_sparse_mla_sink_matches_dense_attention_with_empty_rows_and_paged_cache(
num_heads: int,
block_size: int,
interleaved_pages: bool,
dtype: torch.dtype,
) -> None:
"""Preserve head-specific sinks and latent values across ragged cache pages."""
_require_aiter()
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAHelper
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
ROCMAiterMLASparseImpl,
)
from vllm.v1.attention.backends.mla.sparse_utils import flat_kv_row_view
head_dim, value_dim = 576, 512
generator = torch.Generator().manual_seed(421)
# The empty row exercises sink-only attention; the last row crosses both
# page boundaries and the kernel's 16-key reduction tile.
selected_rows = [
[],
[(2, 7)],
[(3, 1), (0, block_size - 1), (1, 0), (2, 4)],
[(i % 4, (i * 7) % block_size) for i in range(23)],
]
q_cpu = torch.randn(
len(selected_rows), num_heads, head_dim, generator=generator
).to(dtype)
cache_cpu = torch.randn(4, block_size, head_dim, generator=generator).to(dtype)
sinks_cpu = torch.linspace(-4.0, 6.0, num_heads)
# Layers can share a backing allocation, leaving unused rows between pages.
num_layers = 3 if interleaved_pages else 1
backing = torch.full(
(4, num_layers, block_size, head_dim),
float("nan"),
dtype=dtype,
device="cuda",
)
cache = backing[:, num_layers - 1]
cache.copy_(cache_cpu)
indices = [
page * num_layers * block_size + offset
for rows in selected_rows
for page, offset in rows
]
lengths = torch.tensor([0, *(len(rows) for rows in selected_rows)])
longest_row = max(len(rows) for rows in selected_rows)
metadata = SimpleNamespace(
block_size=block_size,
num_prefills=0,
num_decodes=len(selected_rows),
num_decode_tokens=len(selected_rows),
max_query_len=1,
max_seq_len=longest_row,
topk_tokens=longest_row,
qo_indptr=torch.arange(
len(selected_rows) + 1, dtype=torch.int32, device="cuda"
),
paged_kv_last_page_len=torch.ones(
len(selected_rows), dtype=torch.int32, device="cuda"
),
work_meta_data=None,
attn_out_dtype=dtype,
paged_kv_indices=torch.tensor(indices, dtype=torch.int32, device="cuda"),
paged_kv_indptr=lengths.cumsum(0).to(device="cuda", dtype=torch.int32),
)
impl = ROCMAiterMLASparseImpl.__new__(ROCMAiterMLASparseImpl)
impl.num_heads = num_heads
impl.head_size = head_dim
impl.kv_lora_rank = value_dim
impl.scale = head_dim**-0.5
impl.kv_cache_dtype = "auto"
impl.sinks = sinks_cpu.cuda()
padded_q = AiterMLAHelper.get_mla_padded_q(num_heads, q_cpu.cuda())
kv_rows, _ = flat_kv_row_view(cache, block_size)
actual, _ = impl._forward_mla(
SimpleNamespace(_q_scale=None, _k_scale=None),
padded_q,
kv_rows.unsqueeze(1),
metadata,
)
references = []
for query_idx, rows in enumerate(selected_rows):
keys = (
torch.stack([cache_cpu[page, offset] for page, offset in rows])
if rows
else cache_cpu.new_empty((0, head_dim))
)
references.append(
_sink_reference(q_cpu[query_idx], keys, sinks_cpu, impl.scale)
)
expected, _, value_scale, score_error = (
torch.stack(values) for values in zip(*references)
)
assert actual.shape == (len(selected_rows), num_heads, value_dim)
assert actual.dtype == dtype
_assert_sink_output_close(
actual,
expected,
value_scale,
score_error,
dtype,
native=dtype == torch.bfloat16,
)
@pytest.mark.parametrize("layout", ["LBNHC", "LBHNC", "BLNHC"])
def test_sparse_mla_backend_resolves_only_contiguous_layer_layouts(monkeypatch, layout):
from vllm.config import CacheConfig
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
ROCMAiterMLASparseBackend,
)
from vllm.v1.attention.backends.utils import resolve_kv_cache_layout
monkeypatch.setenv("VLLM_KV_CACHE_LAYOUT", layout)
config = SimpleNamespace(cache_config=CacheConfig())
supported = [
[x.name for x in ROCMAiterMLASparseBackend.supported_kv_cache_layouts()]
]
if layout == "BLNHC":
with pytest.raises(ValueError, match="does not satisfy"):
resolve_kv_cache_layout(config, supported)
else:
assert resolve_kv_cache_layout(config, supported).name == layout
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@torch.inference_mode()
def test_sparse_mla_sink_forward_mqa_preserves_split_query(dtype):
"""The public sparse forward joins latent/RoPE queries in the model dtype."""
_require_aiter()
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
ROCMAiterMLASparseImpl,
)
set_random_seed(412)
num_tokens, num_heads, block_size = 2, 8, 16
q = torch.randn(num_tokens, num_heads, Q_HEAD_DIM, device="cuda").to(dtype)
kv = torch.randn(2, 2, block_size, Q_HEAD_DIM, device="cuda").to(dtype)[:, 1]
selected = torch.tensor([[1, 7, 18], [0, 3, 20]], dtype=torch.int32, device="cuda")
sinks = torch.linspace(-3.0, 5.0, num_heads, device="cuda")
impl = object.__new__(ROCMAiterMLASparseImpl)
impl.num_heads = num_heads
impl.kv_lora_rank = V_HEAD_DIM
impl.kv_cache_dtype = "auto"
impl.scale = SM_SCALE
impl.sinks = sinks
impl.topk_indices_buffer = torch.full(
(num_tokens, 128), -1, dtype=torch.int32, device="cuda"
)
impl.topk_indices_buffer[:, : selected.shape[1]] = selected
impl.owns_indexer = True
impl.index_epoch = 0
impl.q_concat_buffer = torch.empty_like(q)
metadata = SimpleNamespace(
attn_out_dtype=dtype,
num_actual_tokens=num_tokens,
num_prefills=0,
num_decodes=num_tokens,
num_decode_tokens=num_tokens,
max_query_len=1,
max_seq_len=kv.shape[0],
block_size=block_size,
topk_tokens=impl.topk_indices_buffer.shape[1],
req_id_per_token=torch.zeros(num_tokens, dtype=torch.int32, device="cuda"),
block_table=torch.tensor([[0, 1]], dtype=torch.int32, device="cuda"),
qo_indptr=torch.arange(num_tokens + 1, dtype=torch.int32, device="cuda"),
paged_kv_indptr=torch.tensor([0, 3, 6], dtype=torch.int32, device="cuda"),
paged_kv_indices=torch.empty(
selected.numel(), dtype=torch.int32, device="cuda"
),
paged_kv_last_page_len=torch.ones(num_tokens, dtype=torch.int32, device="cuda"),
work_meta_data=None,
)
actual, _ = impl.forward_mqa(
(q[..., :V_HEAD_DIM], q[..., V_HEAD_DIM:]),
kv,
metadata,
SimpleNamespace(_q_scale=None, _k_scale=None),
)
references = [
_sink_reference(q[i], kv.flatten(0, 1)[selected[i].long()], sinks, SM_SCALE)
for i in range(num_tokens)
]
expected, _, value_scale, score_error = (
torch.stack(values) for values in zip(*references)
)
assert actual.dtype == dtype
_assert_sink_output_close(
actual,
expected,
value_scale,
score_error,
dtype,
native=dtype == torch.bfloat16,
)
@@ -1,295 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from vllm.platforms import current_platform
if not current_platform.is_rocm():
pytest.skip(
"ROCm AITER sparse MLA index remap test requires ROCm.",
allow_module_level=True,
)
from vllm._aiter_ops import is_aiter_found_and_supported
if not is_aiter_found_and_supported():
pytest.skip(
"ROCm AITER sparse MLA index remap test requires a supported AITER "
"installation.",
allow_module_level=True,
)
from vllm.model_executor.layers.attention import sparse_mla_attention as sparse_mla_mod
from vllm.v1.attention.backends.mla import rocm_aiter_mla_sparse as sparse_mod
NUM_HEADS = 16
KV_LORA_RANK = 512
QK_NOPE_HEAD_DIM = 128
QK_ROPE_HEAD_DIM = 64
HEAD_SIZE = 576
NUM_TOKENS = 2
TOPK_TOKENS = 4
@pytest.fixture(autouse=True)
def _reset_index_epochs():
# The counter is keyed by id(), and a freed test buffer can hand its id to
# the next one, so a leftover count would leak across tests.
sparse_mod._INDEX_EPOCHS.clear()
yield
sparse_mod._INDEX_EPOCHS.clear()
def _make_metadata():
return SimpleNamespace(
num_actual_tokens=NUM_TOKENS,
req_id_per_token=torch.zeros(NUM_TOKENS, dtype=torch.int32, device="cpu"),
block_table=torch.arange(16, dtype=torch.int32, device="cpu").view(2, 8),
paged_kv_indptr=torch.zeros(NUM_TOKENS + 1, dtype=torch.int32, device="cpu"),
paged_kv_indices=torch.zeros(
NUM_TOKENS * TOPK_TOKENS, dtype=torch.int32, device="cpu"
),
block_size=1,
topk_tokens=TOPK_TOKENS,
remapped_buf=None,
remapped_epoch=-1,
)
def _patch_init_deps(monkeypatch, *, speculative: bool = False):
monkeypatch.setattr(
sparse_mod.AiterMLAHelper,
"check_num_heads_validity",
staticmethod(lambda num_heads: None),
)
vllm_config = SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=NUM_TOKENS),
model_config=SimpleNamespace(dtype=torch.bfloat16),
speculative_config=SimpleNamespace() if speculative else None,
attention_config=SimpleNamespace(hisparse_config=None),
)
monkeypatch.setattr(sparse_mod, "get_current_vllm_config", lambda: vllm_config)
monkeypatch.setattr(sparse_mla_mod, "get_current_vllm_config", lambda: vllm_config)
monkeypatch.setattr(
sparse_mla_mod, "get_tensor_model_parallel_world_size", lambda: 1
)
monkeypatch.setattr(
sparse_mod,
"current_workspace_manager",
lambda: SimpleNamespace(
get_simultaneous=lambda *specs: tuple(
torch.zeros(shape, dtype=dtype, device="cpu") for shape, dtype in specs
)
),
)
def _build_impl(indexer, topk_indices_buffer):
return sparse_mod.ROCMAiterMLASparseImpl(
num_heads=NUM_HEADS,
head_size=HEAD_SIZE,
scale=1.0,
num_kv_heads=1,
alibi_slopes=None,
sliding_window=None,
kv_cache_dtype="auto",
logits_soft_cap=None,
attn_type="decoder",
kv_sharing_target_layer_name=None,
topk_indices_buffer=topk_indices_buffer,
indexer=indexer,
q_lora_rank=None,
kv_lora_rank=KV_LORA_RANK,
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
qk_head_dim=QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM,
v_head_dim=QK_NOPE_HEAD_DIM,
kv_b_proj=None,
)
def _new_buffer():
return torch.zeros(NUM_TOKENS, TOPK_TOKENS, dtype=torch.int32, device="cpu")
def _patch_forward_deps(monkeypatch, impl, remap_calls):
monkeypatch.setattr(
sparse_mod,
"triton_convert_req_index_to_global_index",
lambda *args, **kwargs: remap_calls.append(args),
)
monkeypatch.setattr(
sparse_mod.AiterMLAHelper,
"get_mla_padded_q",
staticmethod(lambda num_heads, q: q),
)
attn_out = torch.zeros(NUM_TOKENS, NUM_HEADS, KV_LORA_RANK, device="cpu")
monkeypatch.setattr(
type(impl),
"_forward_mla",
lambda self, layer, q, kv_cache, attn_metadata: (attn_out, None),
)
return attn_out
def _run_forward(impl, metadata):
return impl.forward_mqa(
torch.zeros(NUM_TOKENS, NUM_HEADS, HEAD_SIZE, device="cpu"),
torch.zeros(4, 1, HEAD_SIZE, device="cpu"),
metadata,
SimpleNamespace(_q_scale=None, _k_scale=None),
)
@pytest.mark.parametrize("has_indexer", [True, False])
def test_owns_indexer_is_structural(monkeypatch, has_indexer):
"""owns_indexer reflects the layer, not the serving config."""
_patch_init_deps(monkeypatch)
buffer = _new_buffer()
indexer = SimpleNamespace(topk_indices_buffer=buffer) if has_indexer else None
impl = _build_impl(indexer, buffer)
assert impl.owns_indexer is has_indexer
assert impl.topk_indices_buffer is buffer
def test_forward_mqa_remaps_once_per_metadata(monkeypatch):
"""Skip layers sharing a metadata share its remap; the first one pays.
All layers read one top-k buffer, so the reuse has to key off the metadata
that owns ``paged_kv_indices``, not off the buffer being distinct.
"""
_patch_init_deps(monkeypatch)
buffer = _new_buffer()
first = _build_impl(None, buffer)
second = _build_impl(None, buffer)
remap_calls: list[tuple] = []
attn_out = _patch_forward_deps(monkeypatch, first, remap_calls)
metadata = _make_metadata()
out, lse = _run_forward(first, metadata)
assert out is attn_out
assert lse is None
assert len(remap_calls) == 1
assert metadata.remapped_buf is buffer
_run_forward(second, metadata)
assert len(remap_calls) == 1
def test_forward_mqa_remaps_for_each_metadata(monkeypatch):
"""Skip layers on a second metadata must remap its own paged_kv_indices.
Regression: gating on ``indexer is not None`` skipped every layer in the
non-indexer metadata group, leaving its ``paged_kv_indices`` unwritten.
"""
_patch_init_deps(monkeypatch)
buffer = _new_buffer()
indexer_layer = _build_impl(SimpleNamespace(topk_indices_buffer=buffer), buffer)
skip_layer = _build_impl(None, buffer)
remap_calls: list[tuple] = []
_patch_forward_deps(monkeypatch, indexer_layer, remap_calls)
indexer_metadata = _make_metadata()
_run_forward(indexer_layer, indexer_metadata)
assert len(remap_calls) == 1
assert indexer_metadata.remapped_buf is buffer
# A distinct metadata instance carries its own, still-unwritten indices.
skip_metadata = _make_metadata()
_run_forward(skip_layer, skip_metadata)
assert len(remap_calls) == 2
assert skip_metadata.remapped_buf is buffer
def test_forward_mqa_remaps_after_a_later_indexer_write(monkeypatch):
"""A skip metadata must be refreshed once per indexer write, not once ever.
Regression for cross-layer index sharing (``index_topk_freq``). With
freq=4 on a 78-layer model the 57 skip layers share one metadata while the
21 indexer layers share another, so the skip metadata has to pick up every
new selection. Recording only the buffer left layers 4 onwards attending
to the first selection of the pass, which collapsed RULER niah_single_2 to
0.026 on GLM-5.3 and took GPQA from 0.884 to 0.672.
"""
_patch_init_deps(monkeypatch)
buffer = _new_buffer()
# Construction order mirrors the layer stack: indexer, three skips, indexer.
_build_impl(SimpleNamespace(topk_indices_buffer=buffer), buffer)
first_skip = _build_impl(None, buffer)
reuse_skip = _build_impl(None, buffer)
_build_impl(SimpleNamespace(topk_indices_buffer=buffer), buffer)
next_skip = _build_impl(None, buffer)
remap_calls: list[tuple] = []
_patch_forward_deps(monkeypatch, first_skip, remap_calls)
skip_metadata = _make_metadata()
_run_forward(first_skip, skip_metadata)
assert len(remap_calls) == 1
# Same selection, same metadata: reuse.
_run_forward(reuse_skip, skip_metadata)
assert len(remap_calls) == 1
# A later indexer layer has since rewritten the buffer in place. The object
# is unchanged, so only the epoch can reveal it.
_run_forward(next_skip, skip_metadata)
assert len(remap_calls) == 2
assert skip_metadata.remapped_epoch == next_skip.index_epoch
def test_index_epoch_tracks_the_preceding_indexer_write(monkeypatch):
"""Layers between two indexer layers share the earlier one's epoch."""
_patch_init_deps(monkeypatch)
buffer = _new_buffer()
first_indexer = _build_impl(SimpleNamespace(topk_indices_buffer=buffer), buffer)
skip_a = _build_impl(None, buffer)
skip_b = _build_impl(None, buffer)
second_indexer = _build_impl(SimpleNamespace(topk_indices_buffer=buffer), buffer)
skip_c = _build_impl(None, buffer)
assert skip_a.index_epoch == first_indexer.index_epoch
assert skip_b.index_epoch == first_indexer.index_epoch
assert second_indexer.index_epoch == first_indexer.index_epoch + 1
assert skip_c.index_epoch == second_indexer.index_epoch
def test_forward_mqa_remaps_when_buffer_differs(monkeypatch):
"""A layer reading a different buffer cannot reuse the metadata's remap."""
_patch_init_deps(monkeypatch)
remapped_buffer = _new_buffer()
other_buffer = _new_buffer()
layer = _build_impl(None, other_buffer)
remap_calls: list[tuple] = []
_patch_forward_deps(monkeypatch, layer, remap_calls)
metadata = _make_metadata()
metadata.remapped_buf = remapped_buffer
_run_forward(layer, metadata)
assert len(remap_calls) == 1
assert metadata.remapped_buf is other_buffer
def test_forward_mqa_indexer_layer_always_remaps(monkeypatch):
"""The indexer rewrites its buffer in place, so its remap is never stale."""
_patch_init_deps(monkeypatch)
buffer = _new_buffer()
impl = _build_impl(SimpleNamespace(topk_indices_buffer=buffer), buffer)
remap_calls: list[tuple] = []
_patch_forward_deps(monkeypatch, impl, remap_calls)
metadata = _make_metadata()
metadata.remapped_buf = buffer
metadata.remapped_epoch = impl.index_epoch
_run_forward(impl, metadata)
assert len(remap_calls) == 1
@@ -1,751 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import sys
from types import ModuleType, SimpleNamespace
from unittest.mock import Mock
import pytest
import torch
from vllm.platforms import current_platform
if not current_platform.is_rocm():
pytest.skip(
"ROCm AITER sparse MLA metadata sync test requires ROCm.",
allow_module_level=True,
)
from vllm._aiter_ops import is_aiter_found_and_supported
if not is_aiter_found_and_supported():
pytest.skip(
"ROCm AITER sparse MLA metadata sync test requires a supported AITER "
"installation.",
allow_module_level=True,
)
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backends.mla import rocm_aiter_mla_sparse as sparse_mod
class _FakeAiter(ModuleType):
get_mla_metadata_v1: Mock
@pytest.mark.parametrize("block_size", [16, 64, 128])
@pytest.mark.parametrize("topk_tokens", [128, 256])
def test_rocm_index_conversion_matrix(block_size, topk_tokens):
"""Convert valid, masked, and out-of-range sparse indices exactly."""
device = torch.device("cuda")
num_tokens = 4
max_blocks = 4
req_ids = torch.tensor([0, 1, 0, 1], dtype=torch.int32, device=device)
block_table = torch.tensor(
[[3, 5, 7, 9], [2, 4, 6, 8]], dtype=torch.int32, device=device
)
token_indices = torch.arange(
num_tokens * topk_tokens, dtype=torch.int32, device=device
).view(num_tokens, topk_tokens)
token_indices %= max_blocks * block_size
token_indices[0, 0] = -1
token_indices[1, 1] = max_blocks * block_size
cu_seqlens = torch.arange(
0,
(num_tokens + 1) * topk_tokens,
topk_tokens,
dtype=torch.int32,
device=device,
)
output = torch.full(
(num_tokens * topk_tokens,), -99, dtype=torch.int32, device=device
)
sparse_mod.triton_convert_req_index_to_global_index(
req_ids,
block_table,
token_indices,
cu_seqlens,
output,
BLOCK_STRIDE_ROWS=block_size,
BLOCK_SIZE=block_size,
NUM_TOPK_TOKENS=topk_tokens,
)
block_ids = token_indices // block_size
valid = (token_indices >= 0) & (block_ids < max_blocks)
safe_blocks = block_ids.clamp(0, max_blocks - 1)
bases = block_table[req_ids[:, None].long(), safe_blocks.long()]
expected = torch.where(valid, bases * block_size + token_indices % block_size, 0)
torch.testing.assert_close(output.view_as(expected), expected, rtol=0, atol=0)
def test_rocm_index_conversion_rejects_mismatched_token_extents():
"""Reject full-batch request IDs paired with decode-subset indices."""
with pytest.raises(AssertionError, match="must cover the same tokens"):
sparse_mod.triton_convert_req_index_to_global_index(
torch.zeros(3, dtype=torch.int32),
torch.zeros(1, 4, dtype=torch.int32),
torch.zeros(2, 128, dtype=torch.int32),
torch.arange(3, dtype=torch.int32),
torch.zeros(256, dtype=torch.int32),
BLOCK_STRIDE_ROWS=64,
NUM_TOPK_TOKENS=128,
)
def test_rocm_index_conversion_respects_packed_valid_counts():
"""ROCm's indptr contract packs only each token's valid top-k prefix."""
device = torch.device("cuda")
topk_tokens = 128
valid_counts = torch.tensor([1, 17, 128], dtype=torch.int32, device=device)
cu_seqlens = torch.zeros(4, dtype=torch.int32, device=device)
torch.cumsum(valid_counts, dim=0, out=cu_seqlens[1:])
req_ids = torch.tensor([0, 0, 0], dtype=torch.int32, device=device)
block_table = torch.tensor([[2, 3, 4]], dtype=torch.int32, device=device)
token_indices = torch.arange(
3 * topk_tokens, dtype=torch.int32, device=device
).view(3, topk_tokens)
token_indices %= 3 * 64
output = torch.full((int(cu_seqlens[-1]),), -99, dtype=torch.int32, device=device)
sparse_mod.triton_convert_req_index_to_global_index(
req_ids,
block_table,
token_indices,
cu_seqlens,
output,
BLOCK_STRIDE_ROWS=64,
BLOCK_SIZE=64,
NUM_TOPK_TOKENS=topk_tokens,
)
expected_parts = []
for token_id, count in enumerate(valid_counts.tolist()):
local = token_indices[token_id, :count]
expected_parts.append(block_table[0, (local // 64).long()] * 64 + local % 64)
expected = torch.cat(expected_parts)
torch.testing.assert_close(output, expected, rtol=0, atol=0)
def _make_builder():
builder = object.__new__(sparse_mod.ROCMAiterMLASparseMetadataBuilder)
max_num_batched_tokens = 8
topk_tokens = 4
builder.device = torch.device("cpu")
builder.kv_cache_spec = SimpleNamespace(block_size=1)
builder.model_dtype = torch.bfloat16
builder.kv_cache_dtype = "fp8"
builder.mla_dims = SimpleNamespace(kv_lora_rank=512, qk_rope_head_dim=64)
builder.topk_tokens = topk_tokens
# SparseMLACommonMetadataBuilder state consumed by its build() method.
builder.cp_kv_cache_interleave_size = 1
builder.use_pcp = False
builder._prefill_backend = None
def cpu_token_to_req(metadata):
return torch.repeat_interleave(
torch.arange(metadata.num_reqs, dtype=torch.int32),
metadata.query_start_loc_cpu.diff(),
)
builder._build_req_id_per_token = cpu_token_to_req # type: ignore[method-assign]
builder.req_id_per_token_buffer = torch.zeros(
max_num_batched_tokens, dtype=torch.int32, device="cpu"
)
builder.qo_indptr = torch.arange(
max_num_batched_tokens + 1, dtype=torch.int32, device="cpu"
)
builder.paged_kv_last_page_len = torch.ones(
max_num_batched_tokens, dtype=torch.int32, device="cpu"
)
builder.paged_kv_indices = torch.zeros(
max_num_batched_tokens * topk_tokens, dtype=torch.int32, device="cpu"
)
builder.paged_kv_indptr = torch.zeros(
max_num_batched_tokens + 1, dtype=torch.int32, device="cpu"
)
builder._use_persistent_metadata = True
builder._num_attention_heads = 16
builder._num_compute_units = current_platform.num_compute_units()
builder._mla_q_dtype = torch.bfloat16
builder._mla_kv_dtype = torch.bfloat16
builder._mla_work_meta_data = torch.empty(1, dtype=torch.int32, device="cpu")
builder._mla_work_indptr = torch.empty(1, dtype=torch.int32, device="cpu")
builder._mla_work_info_set = torch.empty(1, dtype=torch.int32, device="cpu")
builder._mla_reduce_indptr = torch.empty(1, dtype=torch.int32, device="cpu")
builder._mla_reduce_final_map = torch.empty(1, dtype=torch.int32, device="cpu")
builder._mla_reduce_partial_map = torch.empty(1, dtype=torch.int32, device="cpu")
builder._prev_req_extent = 0
builder._prev_indices_extent = 0
builder._prev_metadata_key = None
return builder
def _make_common_metadata():
query_start_loc = torch.tensor([0, 1, 2], dtype=torch.int32, device="cpu")
seq_lens = torch.tensor([16, 8], dtype=torch.int32, device="cpu")
return CommonAttentionMetadata(
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc,
seq_lens=seq_lens,
seq_lens_cpu_upper_bound=seq_lens,
num_reqs=2,
num_actual_tokens=2,
max_query_len=1,
max_seq_len=16,
block_table_tensor=torch.arange(16, dtype=torch.int32, device="cpu").view(2, 8),
slot_mapping=torch.arange(2, dtype=torch.int64, device="cpu"),
)
def _make_mixed_common_metadata():
# req0: query_len 1 (decode), req1: query_len 4 (prefill) -> decode-first
query_start_loc = torch.tensor([0, 1, 5], dtype=torch.int32, device="cpu")
seq_lens = torch.tensor([16, 10], dtype=torch.int32, device="cpu")
return CommonAttentionMetadata(
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc,
seq_lens=seq_lens,
seq_lens_cpu_upper_bound=seq_lens,
num_reqs=2,
num_actual_tokens=5,
max_query_len=4,
max_seq_len=16,
block_table_tensor=torch.arange(16, dtype=torch.int32, device="cpu").view(2, 8),
slot_mapping=torch.arange(5, dtype=torch.int64, device="cpu"),
)
def _patch_build_deps(monkeypatch, events=None):
"""Stub the aiter kernel, triton helper and CUDA sync so ``build()`` runs
on CPU."""
def fake_generate_sparse_seqlen_triton(
seq_lens, cu_query_lens, topk_token, num_tokens, max_query_len
):
return torch.zeros(num_tokens, dtype=torch.int32, device="cpu")
fake_aiter = _FakeAiter("aiter")
fake_aiter.get_mla_metadata_v1 = Mock(side_effect=lambda *a, **k: None)
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
monkeypatch.setattr(
sparse_mod, "generate_sparse_seqlen_triton", fake_generate_sparse_seqlen_triton
)
monkeypatch.setattr(
sparse_mod.torch.cuda,
"current_stream",
lambda device=None: SimpleNamespace(
synchronize=lambda: events.append("sync") if events is not None else None
),
)
return fake_aiter
def test_build_populates_decode_only_split_fields(monkeypatch):
"""Decode-only batch: all reqs count as decodes, prefill fields default."""
builder = _make_builder()
_patch_build_deps(monkeypatch)
md = builder.build(
common_prefix_len=0, common_attn_metadata=_make_common_metadata()
)
assert md.num_decodes == 2
assert md.num_prefills == 0
assert md.num_decode_tokens == 2
assert md.prefill_max_seq_len == 0
assert md.prefill is None
def test_build_populates_mixed_split_fields(monkeypatch):
"""Mixed decode+prefill batch: split is reported, prefill fields stay
default because this impl always runs the MQA path."""
builder = _make_builder()
_patch_build_deps(monkeypatch)
md = builder.build(
common_prefix_len=0, common_attn_metadata=_make_mixed_common_metadata()
)
assert md.num_decodes == 1
assert md.num_prefills == 1
assert md.num_decode_tokens == 1
assert md.prefill_max_seq_len == 0
assert md.prefill is None
def test_build_fills_paged_kv_indptr_tail(monkeypatch):
"""Rows past num_tokens must read as empty: the tail repeats the total."""
builder = _make_builder()
_patch_build_deps(monkeypatch)
monkeypatch.setattr(
sparse_mod,
"generate_sparse_seqlen_triton",
lambda *args, **kwargs: torch.tensor([3, 5], dtype=torch.int32),
)
builder.paged_kv_indptr[1:].fill_(-1)
md = builder.build(
common_prefix_len=0, common_attn_metadata=_make_common_metadata()
)
assert md.paged_kv_indptr.tolist() == [0, 3, 8]
assert builder.paged_kv_indptr.tolist() == [0, 3, 8, 8, 8, 8, 8, 8, 8]
def test_sink_build_skips_persistent_metadata(monkeypatch):
builder = _make_builder()
builder._use_persistent_metadata = False
fake_aiter = _patch_build_deps(monkeypatch)
md = builder.build(
common_prefix_len=0, common_attn_metadata=_make_common_metadata()
)
fake_aiter.get_mla_metadata_v1.assert_not_called()
assert md.work_meta_data is None
assert md.work_indptr is None
assert md.work_info_set is None
assert md.reduce_indptr is None
assert md.reduce_final_map is None
assert md.reduce_partial_map is None
assert builder._prev_metadata_key is None
def test_sparse_persistent_metadata_syncs_only_after_recompute(monkeypatch):
builder = _make_builder()
common_metadata = _make_common_metadata()
events: list[str] = []
def fake_generate_sparse_seqlen_triton(*args, **kwargs):
return torch.tensor([1, 2], dtype=torch.int32, device="cpu")
fake_aiter = _FakeAiter("aiter")
def fake_get_mla_metadata_v1(*args, **kwargs):
events.append("metadata")
fake_get_mla_metadata_v1_mock = Mock(side_effect=fake_get_mla_metadata_v1)
fake_aiter.get_mla_metadata_v1 = fake_get_mla_metadata_v1_mock
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
monkeypatch.setattr(
sparse_mod, "generate_sparse_seqlen_triton", fake_generate_sparse_seqlen_triton
)
monkeypatch.setattr(
sparse_mod.torch.cuda,
"current_stream",
lambda device=None: SimpleNamespace(synchronize=lambda: events.append("sync")),
)
builder.build(common_prefix_len=0, common_attn_metadata=common_metadata)
assert events == ["metadata", "sync"]
assert fake_get_mla_metadata_v1_mock.call_count == 1
assert fake_get_mla_metadata_v1_mock.call_args.kwargs["max_split_per_batch"] == 1
assert fake_get_mla_metadata_v1_mock.call_args.kwargs["dtype_q"] == torch.bfloat16
assert fake_get_mla_metadata_v1_mock.call_args.kwargs["dtype_kv"] == torch.bfloat16
events.clear()
builder.build(common_prefix_len=0, common_attn_metadata=common_metadata)
assert events == []
assert fake_get_mla_metadata_v1_mock.call_count == 1
def test_dense_prefill_builds_sparse_metadata_for_decode_subset(monkeypatch):
"""Dense prefill must exclude prefill tokens from AITER sparse metadata.
A mixed batch is routed as sparse MQA for its leading decode requests and
dense MHA for its trailing short prefill requests. The AITER metadata
buffers therefore must describe only the decode subset; including prefill
tokens would mismatch the query passed to ``forward_mqa``.
"""
builder = _make_builder()
builder.vllm_config = SimpleNamespace(
attention_config=SimpleNamespace(sparse_mla_force_mqa=False)
)
query_start_loc = torch.tensor([0, 1, 5], dtype=torch.int32)
seq_lens = torch.tensor([16, 4], dtype=torch.int32)
common_metadata = SimpleNamespace(
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc,
seq_lens=seq_lens,
seq_lens_cpu=seq_lens,
max_query_len=4,
max_seq_len=16,
)
metadata = sparse_mod.ROCMAiterMLASparseMetadata(
num_reqs=2,
max_query_len=4,
max_seq_len=16,
num_actual_tokens=5,
query_start_loc=query_start_loc,
slot_mapping=torch.arange(5),
block_table=torch.arange(16, dtype=torch.int32).view(2, 8),
req_id_per_token=torch.tensor([0, 1, 1, 1, 1], dtype=torch.int32),
topk_tokens=4,
num_decodes=1,
num_prefills=1,
num_decode_tokens=1,
seq_lens=seq_lens,
prefill_max_seq_len=4,
prefill=SimpleNamespace(use_dense_mha=True),
)
monkeypatch.setattr(
sparse_mod.SparseMLACommonMetadataBuilder,
"build",
lambda *args, **kwargs: metadata,
)
sparse_call: dict[str, torch.Tensor] = {}
def fake_generate_sparse_seqlen(
sparse_seq_lens, sparse_query_start_loc, topk, num_tokens, max_q
):
sparse_call.update(
query_lens=torch.diff(sparse_query_start_loc),
seq_lens=sparse_seq_lens.clone(),
query_start_loc=sparse_query_start_loc.clone(),
num_tokens=num_tokens,
)
return torch.ones(num_tokens, dtype=torch.int32)
monkeypatch.setattr(
sparse_mod, "generate_sparse_seqlen_triton", fake_generate_sparse_seqlen
)
fake_aiter = _FakeAiter("aiter")
fake_aiter.get_mla_metadata_v1 = Mock()
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
monkeypatch.setattr(
sparse_mod.torch.cuda,
"current_stream",
lambda device=None: SimpleNamespace(synchronize=lambda: None),
)
result = builder.build(0, common_metadata)
assert result.prefill is metadata.prefill
assert result.num_prefills == 1
assert result.req_id_per_token.tolist() == [0]
assert result.qo_indptr.numel() == 2
assert result.paged_kv_indptr.numel() == 2
assert sparse_call["num_tokens"] == 1
assert sparse_call["query_lens"].tolist() == [1]
assert sparse_call["seq_lens"].tolist() == [16]
assert sparse_call["query_start_loc"].tolist() == [0, 1]
def test_long_prefill_shares_full_sparse_metadata_with_decode(monkeypatch):
"""A prefill longer than top-k must remain in shared sparse metadata.
Dense MHA is only selected when the prefill sequence fits within the
sparse top-k window. Once it exceeds that window, both decode and prefill
are handled by sparse MQA, so every query token must be represented by the
same sparse paging metadata.
"""
builder = _make_builder()
builder.vllm_config = SimpleNamespace(
attention_config=SimpleNamespace(sparse_mla_force_mqa=False)
)
# One decode token followed by a five-token prefill. topk_tokens is four,
# so the prefill must not be split onto the dense-MHA path.
query_start_loc = torch.tensor([0, 1, 6], dtype=torch.int32)
seq_lens = torch.tensor([16, 5], dtype=torch.int32)
common_metadata = SimpleNamespace(
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc,
seq_lens=seq_lens,
seq_lens_cpu=seq_lens,
max_query_len=5,
max_seq_len=16,
)
metadata = sparse_mod.ROCMAiterMLASparseMetadata(
num_reqs=2,
max_query_len=5,
max_seq_len=16,
num_actual_tokens=6,
query_start_loc=query_start_loc,
slot_mapping=torch.arange(6),
block_table=torch.arange(16, dtype=torch.int32).view(2, 8),
req_id_per_token=torch.tensor([0, 1, 1, 1, 1, 1], dtype=torch.int32),
topk_tokens=4,
num_decodes=1,
num_prefills=1,
num_decode_tokens=1,
seq_lens=seq_lens,
prefill_max_seq_len=5,
prefill=SimpleNamespace(use_dense_mha=False),
)
monkeypatch.setattr(
sparse_mod.SparseMLACommonMetadataBuilder,
"build",
lambda *args, **kwargs: metadata,
)
sparse_call: dict[str, torch.Tensor] = {}
def fake_generate_sparse_seqlen(
sparse_seq_lens, sparse_query_start_loc, topk, num_tokens, max_q
):
sparse_call.update(
query_lens=torch.diff(sparse_query_start_loc),
seq_lens=sparse_seq_lens.clone(),
query_start_loc=sparse_query_start_loc.clone(),
num_tokens=num_tokens,
)
return torch.ones(num_tokens, dtype=torch.int32)
monkeypatch.setattr(
sparse_mod, "generate_sparse_seqlen_triton", fake_generate_sparse_seqlen
)
fake_aiter = _FakeAiter("aiter")
get_persistent_metadata = Mock()
fake_aiter.get_mla_metadata_v1 = get_persistent_metadata
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
monkeypatch.setattr(
sparse_mod.torch.cuda,
"current_stream",
lambda device=None: SimpleNamespace(synchronize=lambda: None),
)
result = builder.build(0, common_metadata)
assert result.num_decodes == 1
assert result.num_prefills == 1
assert result.req_id_per_token.tolist() == [0, 1, 1, 1, 1, 1]
assert result.qo_indptr.numel() == 7
assert result.paged_kv_indptr.numel() == 7
assert result.paged_kv_indices.numel() == 6 * result.topk_tokens
assert sparse_call["num_tokens"] == 6
assert sparse_call["query_lens"].tolist() == [1, 5]
assert sparse_call["seq_lens"].tolist() == [16, 5]
assert sparse_call["query_start_loc"].tolist() == [0, 1, 6]
get_persistent_metadata.assert_called_once()
assert result.work_meta_data is builder._mla_work_meta_data
assert result.reduce_indptr is builder._mla_reduce_indptr
def test_forward_mqa_slices_full_batch_metadata_to_query_tokens(monkeypatch):
"""The sparse kernel inputs must have the same token extent as its query.
In a mixed dense-prefill batch, ``forward_mqa`` receives only decode query
tokens while some metadata buffers may represent the full batch. This
guards the slicing that prevents the index conversion from writing beyond
the decode-sized output.
"""
impl = object.__new__(sparse_mod.ROCMAiterMLASparseImpl)
impl.kv_cache_dtype = "auto"
impl.num_heads = 4
impl.topk_indices_buffer = torch.arange(48, dtype=torch.int32).view(12, 4)
impl.owns_indexer = True
impl.index_epoch = 0
expected_output = torch.zeros(2, 4, 512, dtype=torch.bfloat16)
forward_mla = Mock(return_value=(expected_output, None))
monkeypatch.setattr(impl, "_forward_mla", forward_mla)
metadata = SimpleNamespace(
req_id_per_token=torch.tensor([0, 1] + [2] * 5 + [3] * 5, dtype=torch.int32),
block_table=torch.arange(40, dtype=torch.int32).view(4, 10),
paged_kv_indptr=torch.arange(3, dtype=torch.int32),
paged_kv_indices=torch.zeros(8, dtype=torch.int32),
block_size=1,
topk_tokens=4,
)
query = torch.zeros(2, 4, 576, dtype=torch.bfloat16)
captured = {}
def fake_convert(req_ids, block_table, token_indices, *args, **kwargs):
captured["req_ids"] = req_ids.clone()
captured["token_indices"] = token_indices.clone()
monkeypatch.setattr(
sparse_mod, "triton_convert_req_index_to_global_index", fake_convert
)
monkeypatch.setattr(
sparse_mod.AiterMLAHelper, "get_mla_padded_q", lambda num_heads, q: q
)
output, lse = impl.forward_mqa(
query,
torch.zeros(1, 1, 576, dtype=torch.bfloat16),
metadata,
SimpleNamespace(),
)
assert output is expected_output
assert lse is None
assert captured["req_ids"].tolist() == [0, 1]
assert captured["token_indices"].shape == (2, 4)
forward_mla.assert_called_once()
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
def test_forward_mqa_slices_decode_subset_for_bf16_and_fp8(monkeypatch, kv_cache_dtype):
"""Both cache dtypes must slice mixed-batch metadata to decode queries."""
impl = object.__new__(sparse_mod.ROCMAiterMLASparseImpl)
impl.kv_cache_dtype = kv_cache_dtype
impl.num_heads = 4
impl.topk_indices_buffer = torch.arange(48, dtype=torch.int32).view(12, 4)
impl.owns_indexer = True
impl.index_epoch = 0
expected_output = torch.zeros(2, 4, 512, dtype=torch.bfloat16)
forward_mla = Mock(return_value=(expected_output, None))
monkeypatch.setattr(impl, "_forward_mla", forward_mla)
metadata = SimpleNamespace(
req_id_per_token=torch.tensor([0, 1] + [2] * 5 + [3] * 5, dtype=torch.int32),
block_table=torch.arange(40, dtype=torch.int32).view(4, 10),
paged_kv_indptr=torch.arange(3, dtype=torch.int32),
paged_kv_indices=torch.zeros(8, dtype=torch.int32),
block_size=1,
topk_tokens=4,
)
query = torch.zeros(2, 4, 576, dtype=torch.bfloat16)
kv_cache = torch.zeros(1, 1, 576, dtype=torch.uint8)
captured = {}
def fake_convert(req_ids, block_table, token_indices, *args, **kwargs):
captured["req_ids"] = req_ids.clone()
captured["token_indices"] = token_indices.clone()
monkeypatch.setattr(
sparse_mod, "triton_convert_req_index_to_global_index", fake_convert
)
monkeypatch.setattr(
sparse_mod.AiterMLAHelper, "get_mla_padded_q", lambda num_heads, q: q
)
if kv_cache_dtype == "fp8":
fp8_dtype = current_platform.fp8_dtype()
monkeypatch.setattr(
sparse_mod.ops,
"scaled_fp8_quant",
lambda q, scale: (q.to(fp8_dtype), None),
)
output, lse = impl.forward_mqa(
query,
kv_cache,
metadata,
SimpleNamespace(_q_scale=torch.tensor(1.0)),
)
assert output is expected_output
assert lse is None
assert captured["req_ids"].tolist() == [0, 1]
assert captured["token_indices"].shape == (2, 4)
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
@pytest.mark.parametrize("length_relation", ["short", "long"])
@pytest.mark.parametrize("workload", ["prefill", "mixed", "decode"])
def test_sparse_metadata_workload_length_dtype_matrix(
monkeypatch, workload, length_relation, kv_cache_dtype
):
"""Cover routing and persistent metadata across the ROCm support matrix.
The matrix crosses pure prefill, mixed prefill/decode, and decode-only
workloads with sequence lengths below/above top-k and BF16/FP8 KV cache.
"""
builder = _make_builder()
builder.vllm_config = SimpleNamespace(
attention_config=SimpleNamespace(sparse_mla_force_mqa=False)
)
if kv_cache_dtype == "fp8":
builder._mla_q_dtype = current_platform.fp8_dtype()
builder._mla_kv_dtype = current_platform.fp8_dtype()
is_long = length_relation == "long"
prefill_len = 5 if is_long else 3
decode_seq_len = 8 if is_long else 3
if workload == "prefill":
query_lens = [prefill_len]
seq_lens_list = [prefill_len]
num_decodes, num_prefills = 0, 1
elif workload == "mixed":
query_lens = [1, prefill_len]
seq_lens_list = [decode_seq_len, prefill_len]
num_decodes, num_prefills = 1, 1
else:
query_lens = [1, 1]
seq_lens_list = [decode_seq_len, decode_seq_len + 1]
num_decodes, num_prefills = 2, 0
query_start_loc = torch.tensor(
[0, *torch.tensor(query_lens).cumsum(0).tolist()], dtype=torch.int32
)
seq_lens = torch.tensor(seq_lens_list, dtype=torch.int32)
num_tokens = sum(query_lens)
common_metadata = SimpleNamespace(
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc,
seq_lens=seq_lens,
seq_lens_cpu=seq_lens,
max_query_len=max(query_lens),
max_seq_len=max(seq_lens_list),
)
metadata = sparse_mod.ROCMAiterMLASparseMetadata(
num_reqs=len(query_lens),
max_query_len=max(query_lens),
max_seq_len=max(seq_lens_list),
num_actual_tokens=num_tokens,
query_start_loc=query_start_loc,
slot_mapping=torch.arange(num_tokens),
block_table=torch.arange(len(query_lens) * 8, dtype=torch.int32).view(
len(query_lens), 8
),
req_id_per_token=torch.repeat_interleave(
torch.arange(len(query_lens), dtype=torch.int32),
torch.tensor(query_lens),
),
topk_tokens=4,
num_decodes=num_decodes,
num_prefills=num_prefills,
num_decode_tokens=num_decodes,
seq_lens=seq_lens,
prefill_max_seq_len=prefill_len if num_prefills else 0,
prefill=SimpleNamespace(use_dense_mha=not is_long) if num_prefills else None,
)
monkeypatch.setattr(
sparse_mod.SparseMLACommonMetadataBuilder,
"build",
lambda *args, **kwargs: metadata,
)
monkeypatch.setattr(
sparse_mod,
"generate_sparse_seqlen_triton",
lambda *args, **kwargs: torch.ones(args[3], dtype=torch.int32),
)
get_persistent_metadata = Mock()
fake_aiter = _FakeAiter("aiter")
fake_aiter.get_mla_metadata_v1 = get_persistent_metadata
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
monkeypatch.setattr(
sparse_mod.torch.cuda,
"current_stream",
lambda device=None: SimpleNamespace(synchronize=lambda: None),
)
result = builder.build(0, common_metadata)
uses_dense_prefill = workload in ("prefill", "mixed") and not is_long
expected_sparse_tokens = num_decodes if uses_dense_prefill else num_tokens
assert result.req_id_per_token.numel() == expected_sparse_tokens
assert result.qo_indptr.numel() == expected_sparse_tokens + 1
assert result.paged_kv_indptr.numel() == expected_sparse_tokens + 1
uses_fallback = workload == "prefill" and is_long and kv_cache_dtype == "auto"
if expected_sparse_tokens == 0 or uses_fallback:
get_persistent_metadata.assert_not_called()
else:
get_persistent_metadata.assert_called_once()
if uses_fallback or expected_sparse_tokens == 0:
assert result.work_meta_data is None
assert result.reduce_indptr is None
else:
assert result.work_meta_data is builder._mla_work_meta_data
assert result.reduce_indptr is builder._mla_reduce_indptr
@@ -1,600 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DeepSeek V4.1's ROCm MXFP4 sparse indexer against a torch reference.
The indexer K cache is a view into vLLM's block-major KV pool: its pages are
a block stride apart with other layers' pages in between, so the tests put
it there. The reference reads the keys back from a second cache the same
writer fills in natural order, so it shares none of the preshuffled
addressing it checks.
"""
import itertools
import types
import pytest
import torch
from vllm.platforms import current_platform
if not current_platform.is_rocm():
pytest.skip("ROCm-only", allow_module_level=True)
from vllm.config import CUDAGraphMode
from vllm.models.deepseek_v41.amd.rocm import (
rocm_mxfp4_indexer_k_store,
rocm_mxfp4_indexer_q_quant,
)
from vllm.platforms.rocm import get_cdna_version
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.backends.mla.indexer import (
DeepseekV32IndexerPrefillChunkMetadata,
)
from vllm.v1.attention.backends.mla.rocm_paged_mxfp4_indexer import (
RocmMxfp4IndexerMetadata,
native_decode,
plan_gather_launches,
plan_prefill_chunks,
)
from vllm.v1.attention.ops import rocm_paged_mxfp4_indexer as ops
from vllm.v1.worker.workspace import init_workspace_manager, reset_workspace_manager
if get_cdna_version() != 4:
pytest.skip("CDNA4-only", allow_module_level=True)
HEADS, HEAD_DIM = 32, 128
WIDTH = HEAD_DIM // 2 + HEAD_DIM // 32
# V4.1's 8-token blocks, with the pool (2048 blocks) and top-k (512) scaled
# down so short test contexts still select.
CAND_BLOCK, POOL_BLOCKS, TOPK = 8, 16, 32
MAX_LEN = 4096
DEVICE = "cuda"
_E2M1 = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0])
E2M1 = torch.cat([_E2M1, -_E2M1])
def _dequant(packed, e8m0):
"""[..., D/2] e2m1 pairs and [..., D/32] e8m0 to [..., D] fp32."""
table = E2M1.to(packed.device)
vals = torch.stack([table[(packed & 0xF).long()], table[(packed >> 4).long()]], -1)
scale = torch.pow(2.0, e8m0.float() - 127)
return (vals.flatten(-2).unflatten(-1, (-1, 32)) * scale[..., None]).flatten(-2)
@pytest.fixture(autouse=True)
def _workspace():
init_workspace_manager(torch.device(DEVICE))
yield
reset_workspace_manager()
class _Case:
"""Requests with their keys written into a block-major pool (ratio-2 and
ratio-1 pages side by side in every block) and into natural-order caches
the reference reads. The bf16 inputs are kept for the end-to-end test."""
def __init__(self, seq_lens, block):
from aiter.ops.triton.fusions.k_norm_rope_mxfp4_cache import (
k_norm_rope_mxfp4_cache,
)
self.seq_lens = seq_lens
num_blocks = sum(cdiv(n, block) for n in seq_lens) + 3
perm = torch.randperm(num_blocks)
width = max(cdiv(n, block) for n in seq_lens)
self.block_table = torch.zeros(len(seq_lens), width, dtype=torch.int32)
used = 0
for req, n in enumerate(seq_lens):
self.block_table[req, : cdiv(n, block)] = perm[used : used + cdiv(n, block)]
used += cdiv(n, block)
self.block_table = self.block_table.to(DEVICE)
def align(n):
return cdiv(n, 576) * 576
page2, page1 = block // 2 * WIDTH, block * WIDTH
stride = align(page2) + align(page1) + 576
self.pool = torch.zeros(num_blocks * stride, dtype=torch.uint8, device=DEVICE)
self.page_bytes = torch.zeros_like(self.pool, dtype=torch.bool)
for i in range(num_blocks):
base = i * stride
self.page_bytes[base : base + page2] = True
base += align(page2)
self.page_bytes[base : base + page1] = True
self.cache = {
2: torch.as_strided(
self.pool, (num_blocks, block // 2, WIDTH), (stride, WIDTH, 1), 0
),
1: torch.as_strided(
self.pool,
(num_blocks, block, WIDTH),
(stride, WIDTH, 1),
align(page2),
),
}
self.offsets = [0]
for n in seq_lens:
self.offsets.append(self.offsets[-1] + n)
# the same keys in natural order, one entry per page
self.natural = {
r: torch.zeros(self.offsets[-1], 1, WIDTH, dtype=torch.uint8, device=DEVICE)
for r in (1, 2)
}
self.cos_sin = cos_sin = torch.randn(MAX_LEN, 64, device=DEVICE)
self.norm = norm = torch.rand(HEAD_DIM, device=DEVICE) + 0.5
self.k_pre = {}
for req, n in enumerate(seq_lens):
if n == 0:
continue
k_pre = torch.randn(n, HEAD_DIM, device=DEVICE).to(torch.bfloat16)
self.k_pre[req] = k_pre
pos = torch.arange(n, device=DEVICE)
for ratio, cache in self.cache.items():
comp = pos // ratio
boundary = (pos + 1) % ratio == 0
entries = cache.shape[1]
page = self.block_table[req, comp // entries].long()
slot = torch.where(boundary, page * entries + comp % entries, -1)
nat = torch.where(boundary, self.offsets[req] + comp, -1)
rocm_mxfp4_indexer_k_store(
k_pre,
pos,
cos_sin,
norm,
1e-6,
cache,
slot,
ratio,
True,
num_heads=HEADS,
)
k_norm_rope_mxfp4_cache(
k_pre, pos, cos_sin, norm, 1e-6, self.natural[ratio], nat, ratio
)
def keys(self, ratio, req, n):
nat = self.natural[ratio][self.offsets[req] : self.offsets[req] + n, 0]
return _dequant(nat[:, : HEAD_DIM // 2], nat[:, HEAD_DIM // 2 :])
def _queries(num_rows):
q = torch.randint(0, 256, (num_rows, HEADS, HEAD_DIM // 2), dtype=torch.uint8)
scales = torch.randint(122, 130, (num_rows, HEADS, HEAD_DIM // 32))
q, scales = q.to(DEVICE), scales.to(torch.uint8).to(DEVICE)
weights = torch.randn(num_rows, HEADS, device=DEVICE)
# the fused Q kernel returns the scales as one int32 per head
return q, scales.view(torch.int32).squeeze(-1), weights, _dequant(q, scales)
def _scores(q, w, keys):
return (torch.relu(q @ keys.T) * w[:, None]).sum(0)
def _assert_topk(row, scores, k):
"""``row`` is a top-k of the entries of ``scores`` above -inf, up to the
fp32 reduce order."""
valid = scores > float("-inf")
want = min(k, int(valid.sum()))
sel = row[row >= 0].long()
assert sel.numel() == want and sel.unique().numel() == want
if want:
assert bool(valid[sel].all())
kth = scores[valid].topk(want).values[-1]
finite = scores[torch.isfinite(scores)]
tol = 1e-4 * float(finite.abs().max()) if finite.numel() else 0.0
assert bool((scores[sel] >= kth - tol).all())
def _block_maxima(scores):
end = scores.numel()
if end == 0:
return scores
padded = torch.full((cdiv(end, CAND_BLOCK) * CAND_BLOCK,), float("-inf"))
padded[:end] = scores.cpu()
out = padded.view(-1, CAND_BLOCK).amax(1)
out[(end - 1) // CAND_BLOCK] = float("inf")
return out.to(scores.device)
def _forward_context(monkeypatch, metadata):
context = types.SimpleNamespace(
attn_metadata={"indexer": metadata},
cudagraph_runtime_mode=CUDAGraphMode.NONE,
)
monkeypatch.setattr(ops, "get_forward_context", lambda: context)
def _decode_metadata(case, rows, ratio, query_lens):
lens = torch.tensor([(p + 1) // ratio for _, p in rows], dtype=torch.int32)
lens = lens.to(DEVICE)
block_table = case.block_table[[req for req, _ in rows]].contiguous()
entries = case.cache[ratio].shape[1]
next_n = max(query_lens)
context_lens = (torch.tensor(case.seq_lens, dtype=torch.int32) // ratio).to(DEVICE)
native = native_decode(lens, block_table, query_lens, next_n, context_lens)
schedule = torch.empty(
ops.rocm_mxfp4_decode_schedule_words(HEADS, HEAD_DIM, entries, next_n),
dtype=torch.int32,
device=DEVICE,
)
return RocmMxfp4IndexerMetadata(
seq_lens=None,
max_seq_len=max(case.seq_lens),
slot_mapping=None,
num_decodes=len(case.seq_lens),
num_decode_tokens=len(rows),
num_prefills=0,
num_prefill_tokens=0,
decode=types.SimpleNamespace(block_table=block_table, seq_lens=lens[:, None]),
decode_row_lens=lens,
decode_block_ends=(lens + CAND_BLOCK - 1) // CAND_BLOCK,
decode_native=native,
decode_schedule=ops.build_rocm_mxfp4_decode_schedule(
lens, HEADS, HEAD_DIM, entries, schedule, MAX_LEN // ratio, native
),
)
def _prefill_metadata(
case, rows, ratio, chunk_bounds, query_start_loc, gather_rows, gather
):
chunks = []
for req_lo, req_hi, t0, t1 in chunk_bounds:
ends = torch.tensor(
[(rows[t][1] + 1) // ratio for t in range(t0, t1)], dtype=torch.int32
).to(DEVICE)
chunks.append(
DeepseekV32IndexerPrefillChunkMetadata(
block_table=case.block_table[req_lo:req_hi],
cu_seqlen_ks=torch.zeros_like(ends),
cu_seqlen_ke=ends,
cu_seq_lens=torch.zeros(1, dtype=torch.int32, device=DEVICE),
token_to_seq=torch.zeros(1, dtype=torch.int32, device=DEVICE),
total_seq_lens=1,
max_seqlen_k=int(ends.max()),
token_start=t0,
token_end=t1,
num_reqs=req_hi - req_lo,
)
)
seq_lens = torch.tensor(case.seq_lens)
context_lens = (seq_lens // ratio).int().to(DEVICE)
plans = plan_prefill_chunks(
chunks,
query_start_loc,
seq_lens,
context_lens,
ratio,
CAND_BLOCK,
0.0 if gather else None,
torch.tensor(query_start_loc, dtype=torch.int32, device=DEVICE),
)
return RocmMxfp4IndexerMetadata(
seq_lens=None,
max_seq_len=max(case.seq_lens),
slot_mapping=None,
num_decodes=0,
num_decode_tokens=0,
num_prefills=len(case.seq_lens),
num_prefill_tokens=len(rows),
prefill=types.SimpleNamespace(chunks=chunks),
prefill_plans=plans,
gather_launches=plan_gather_launches(chunks, plans, gather_rows),
)
# The kernel prefers 128-token pages.
BLOCKS = pytest.mark.parametrize("block", [128, 64])
@BLOCKS
def test_k_store_stays_inside_its_pages(block):
"""The writer touches only its pages of the block-major pool. What they hold
is checked by the layer tests, which read them back through aiter's kernel."""
set_random_seed(0)
case = _Case([700, 333], block)
assert int(case.pool[~case.page_bytes].count_nonzero()) == 0
@pytest.mark.parametrize(
"rows, width, k",
# vLLM's kernel below 128 rows, aiter's from 128 rows and on every k=2048
[(8, 4096, 512), (128, 4096, 512), (8, 16384, 2048)],
)
def test_topk_matches_torch_on_both_kernels(rows, width, k):
set_random_seed(0)
logits = torch.randn(rows, width, device=DEVICE)
lengths = torch.randint(1, width + 1, (rows,), dtype=torch.int32, device=DEVICE)
lengths[0] = k // 2 # shorter than k, so the rest of the row pads with -1
out = torch.full((rows, k), 7, dtype=torch.int32, device=DEVICE)
ops._topk(logits, lengths, out, k)
for r in range(rows):
n = min(k, int(lengths[r]))
want = torch.topk(logits[r, : int(lengths[r])], n).indices.sort().values
got = out[r]
assert torch.equal(got[got >= 0].sort().values, want.int())
assert int((got == -1).sum()) == k - n
def _run_layers(monkeypatch, case, rows, metadata):
"""Ratio-2 dense layer, the candidate source, and a consumer both ways;
each checked against the torch reference."""
num_rows = len(rows)
q, q_scale, weights, q_ref = _queries(num_rows)
hidden = torch.empty(num_rows, 1, device=DEVICE)
def topk_buffer():
return torch.full((num_rows + 2, TOPK), 7, dtype=torch.int32, device=DEVICE)
_forward_context(monkeypatch, metadata(2))
out = topk_buffer()
ops.rocm_mxfp4_sparse_attn_indexer(
hidden,
"indexer",
case.cache[2],
q,
q_scale,
weights,
TOPK,
HEAD_DIM,
MAX_LEN // 2,
out,
compress_ratio=2,
)
for i, (req, pos) in enumerate(rows):
end = (pos + 1) // 2
keys = case.keys(2, req, end)
_assert_topk(out[i], _scores(q_ref[i], weights[i], keys), TOPK)
source = metadata(1)
_forward_context(monkeypatch, source)
pool = torch.full((num_rows + 2, POOL_BLOCKS), 5, dtype=torch.int32, device=DEVICE)
out = topk_buffer()
ops.rocm_mxfp4_sparse_attn_indexer(
hidden,
"indexer",
case.cache[1],
q,
q_scale,
weights,
TOPK,
HEAD_DIM,
MAX_LEN,
out,
compress_ratio=1,
candidate_blocks=pool,
candidate_block_size=CAND_BLOCK,
candidate_write=True,
)
allowed = []
for i, (req, pos) in enumerate(rows):
scores = _scores(q_ref[i], weights[i], case.keys(1, req, pos + 1))
_assert_topk(out[i], scores, TOPK)
_assert_topk(pool[i], _block_maxima(scores), POOL_BLOCKS)
mask = torch.zeros_like(scores, dtype=torch.bool)
for block in pool[i][pool[i] >= 0].tolist():
mask[block * CAND_BLOCK : (block + 1) * CAND_BLOCK] = True
allowed.append(torch.where(mask, scores, float("-inf")))
pa = ops._aiter()
build = pa.build_candidate_gather
builds = []
def counting_build(*args, **kwargs):
builds.append(1)
return build(*args, **kwargs)
monkeypatch.setattr(pa, "build_candidate_gather", counting_build)
for gather in (True, False):
consumer = metadata(1, gather)
consumer.decode_use_gather = gather
_forward_context(monkeypatch, consumer)
builds.clear()
# Two consumer layers of one step: the second reuses the first's pools.
for _ in range(2):
out = topk_buffer()
ops.rocm_mxfp4_sparse_mqa_indexer(
hidden,
"indexer",
case.cache[1],
q,
q_scale,
weights,
TOPK,
HEAD_DIM,
MAX_LEN,
out,
1,
pool,
CAND_BLOCK,
POOL_BLOCKS * CAND_BLOCK,
)
for i in range(num_rows):
_assert_topk(out[i], allowed[i], TOPK)
if gather:
launches = len(consumer.gather_launches) + (consumer.decode is not None)
assert len(builds) == launches
@BLOCKS
@pytest.mark.parametrize(
"query_lens",
# Uniform steps launch the dense layers on next_n-row sequences, also with
# cudagraph padding (query length 0) after them; a ragged step keeps a row
# per token.
[[6, 6, 6, 6], [2, 2, 2], [2, 2, 2, 0], [2, 1, 2]],
ids=["native6", "native2", "padded", "ragged"],
)
def test_decode_layers_match_reference(monkeypatch, block, query_lens):
set_random_seed(0)
next_n = max(query_lens)
case = _Case([q and n for q, n in zip(query_lens, [900, 333, 610, 1200])], block)
# a padding request has next_n rows of its own and no context
rows = [
(req, n - q + j if q else -1)
for req, (n, q) in enumerate(zip(case.seq_lens, query_lens))
for j in range(q or next_n)
]
_run_layers(
monkeypatch,
case,
rows,
lambda r, gather=False: _decode_metadata(case, rows, r, query_lens),
)
@pytest.mark.parametrize(
"seq_lens,new_tokens,chunks",
[
# one request past a cached prefix, then one sliced into two chunks
([760, 500], [300, 500], [(0, 1, 0, 300), (1, 2, 300, 560), (1, 2, 560, 800)]),
# three requests in one chunk, the first two with equal rows
([600, 450, 900], [200, 200, 120], [(0, 3, 0, 520)]),
# four requests with four different query lengths in one chunk
([900, 600, 450, 700], [150, 90, 200, 60], [(0, 4, 0, 500)]),
],
ids=["sliced", "batched", "ragged"],
)
# The consumers rejoin a request's rows across chunks, then cut them at most
# gather_rows a launch.
@pytest.mark.parametrize("gather_rows", [1 << 20, 128], ids=["joined", "cut"])
@BLOCKS
def test_prefill_layers_match_reference(
monkeypatch, seq_lens, new_tokens, chunks, gather_rows, block
):
set_random_seed(0)
case = _Case(seq_lens, block)
rows = [
(req, n - q + i)
for req, (n, q) in enumerate(zip(seq_lens, new_tokens))
for i in range(q)
]
query_start_loc = [0, *itertools.accumulate(new_tokens)]
_run_layers(
monkeypatch,
case,
rows,
lambda r, gather=True: _prefill_metadata(
case, rows, r, chunks, query_start_loc, gather_rows, gather
),
)
def _rope_gptj(x, cos_sin, positions):
"""GPT-J RoPE on the last 64 dims, as the indexer rotates Q and K: bf16 in
and out."""
x = x.bfloat16().float()
cos, sin = cos_sin[positions].chunk(2, dim=-1)
if x.dim() == 3:
cos, sin = cos[:, None], sin[:, None]
rot = x[..., -64:]
even, odd = rot[..., 0::2].clone(), rot[..., 1::2].clone()
rot[..., 0::2] = even * cos - odd * sin
rot[..., 1::2] = odd * cos + even * sin
return x.bfloat16()
def _mxfp4_round_trip(x):
"""MXFP4 round trip of x, rounded as the writers do: a scale of
2^ceil(log2(amax / 6)) per 32 values, then the nearest e2m1 value, ties to
even."""
blocks = x.float().unflatten(-1, (-1, 32))
amax = blocks.abs().amax(-1, keepdim=True).clamp(min=2**-126)
scale = torch.exp2(torch.ceil(torch.log2(amax / 6)))
y = (blocks / scale).clamp(-6, 6)
# Even codes first: argmin keeps the first of two equal distances.
grid = torch.cat([_E2M1[0::2], _E2M1[1::2]]).to(x.device)
idx = (y.abs()[..., None] - grid).abs().argmin(-1)
return (grid[idx] * y.sign() * scale).flatten(-2)
def _torch_keys(case, ratio, req):
"""A request's compressed keys in torch: k_norm of each group's last
token, rotated at the group's first position, through MXFP4."""
k = case.k_pre[req][ratio - 1 :: ratio].float()
k = k * torch.rsqrt(k.pow(2).mean(-1, keepdim=True) + 1e-6) * case.norm
pos = torch.arange(k.shape[0], device=DEVICE) * ratio
return _mxfp4_round_trip(_rope_gptj(k, case.cos_sin, pos))
def _topk_indices(q, weights, keys, ends, k):
"""Top-k of sum_h relu(q_h . key) * weights_h over each row's [0, end)."""
scores = (torch.einsum("thd,cd->thc", q, keys).relu() * weights[..., None]).sum(1)
cols = torch.arange(keys.shape[0], device=DEVICE)
return scores.masked_fill(cols >= ends[:, None], float("-inf")).topk(k).indices
def _recall(selected, reference):
"""Per row, the fraction of the reference top-k that ``selected`` picked."""
return (selected[:, :, None] == reference[:, None, :]).any(1).float().mean(1)
@BLOCKS
@pytest.mark.parametrize("ratio", [2, 1])
@pytest.mark.parametrize("step", ["decode", "prefill"])
def test_selection_from_bf16_matches_torch(monkeypatch, step, ratio, block):
"""End to end from bf16 K and Q: vLLM's MXFP4 K store and Q quant, the
paged pool, metadata from the planning functions the builder calls, and
the indexer op pick what the same MXFP4 math picks in torch."""
set_random_seed(0)
if step == "decode":
case = _Case([3000, 2600, 1900, 3500, 1200, 2200, 3900, 1600], block)
rows = [(req, n - 1) for req, n in enumerate(case.seq_lens)]
metadata = _decode_metadata(case, rows, ratio, [1] * len(rows))
else:
new_tokens = [300, 200, 250]
case = _Case([1500, 900, 2000], block)
rows = [
(req, n - q + i)
for req, (n, q) in enumerate(zip(case.seq_lens, new_tokens))
for i in range(q)
]
metadata = _prefill_metadata(
case,
rows,
ratio,
[(0, len(new_tokens), 0, len(rows))],
[0, *itertools.accumulate(new_tokens)],
1 << 20,
gather=False,
)
num_rows = len(rows)
q_bf16 = torch.randn(num_rows, HEADS, HEAD_DIM, device=DEVICE).to(torch.bfloat16)
w = torch.randn(num_rows, HEADS, device=DEVICE)
pos = torch.tensor([p for _, p in rows], device=DEVICE)
scales = (HEAD_DIM**-0.5, HEADS**-0.5)
(q, q_scale), weights = rocm_mxfp4_indexer_q_quant(
pos, q_bf16, case.cos_sin, w, *scales
)
_forward_context(monkeypatch, metadata)
out = torch.full((num_rows, TOPK), 7, dtype=torch.int32, device=DEVICE)
ops.rocm_mxfp4_sparse_attn_indexer(
torch.empty(num_rows, 1, device=DEVICE),
"indexer",
case.cache[ratio],
q,
q_scale,
weights,
TOPK,
HEAD_DIM,
MAX_LEN // ratio,
out,
compress_ratio=ratio,
)
ends = (pos + 1) // ratio
assert bool(((out >= 0) & (out < ends[:, None])).all())
assert all(row.unique().numel() == TOPK for row in out)
q_ref = _mxfp4_round_trip(_rope_gptj(q_bf16, case.cos_sin, pos))
w_ref = w * scales[0] * scales[1]
agree = []
for req in range(len(case.seq_lens)):
mine = torch.tensor([r == req for r, _ in rows], device=DEVICE)
ref = _topk_indices(
q_ref[mine], w_ref[mine], _torch_keys(case, ratio, req), ends[mine], TOPK
)
agree.append(_recall(out[mine], ref))
assert torch.cat(agree).mean().item() >= 0.99
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -1,215 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the multi-token decode flatten in ``TritonMLAImpl.forward_mqa``.
``decode_attention_fwd`` has no causal flag: it launches one program per row of
``q`` and reads that row's KV extent from ``B_Seqlen``. So a ``query_len``-token
decode block is flattened to one row per query token, and intra-block causality
has to be expressed through the per-row sequence lengths -- row ``t`` gets
``seq_len - (query_len - 1) + t``, i.e. the committed prefix plus block tokens
``0..t``. A non-causal draft block instead gives every row the same full extent.
The kernel is replaced by a spy, so these tests assert on the ``block_table``
and ``seq_lens`` the backend actually submits and need no GPU or Triton runtime.
"""
from types import SimpleNamespace
from unittest.mock import patch
import pytest
import torch
from vllm.model_executor.layers.attention.mla_attention import QueryLenSupport
from vllm.v1.attention.backend import AttentionCGSupport
from vllm.v1.attention.backends.mla.triton_mla import (
TritonMLAImpl,
TritonMLAMetadataBuilder,
)
NUM_HEADS = 16
KV_LORA_RANK = 512
QK_ROPE_HEAD_DIM = 64
HEAD_SIZE = KV_LORA_RANK + QK_ROPE_HEAD_DIM
PAGE_SIZE = 16
# Committed context per request, including a fresh request at zero context
# where the whole KV extent is the block itself. A decode block's seq_len
# already counts the block, so seq_len = context + query_len.
CONTEXT_LENS = [0, 1, 37, 512]
def _seq_lens(query_len: int) -> list[int]:
return [c + query_len for c in CONTEXT_LENS]
def _make_impl(dcp_world_size: int = 1) -> TritonMLAImpl:
"""Only the attributes ``forward_mqa`` reads; __init__ wants a full config."""
impl = object.__new__(TritonMLAImpl)
impl.kv_lora_rank = KV_LORA_RANK
impl.scale = HEAD_SIZE**-0.5
impl._sm_count = 304
impl.dcp_world_size = dcp_world_size
return impl
def _make_metadata(seq_lens: list[int], query_len: int, causal: bool):
num_decodes = len(seq_lens)
return SimpleNamespace(
num_decodes=num_decodes,
num_decode_tokens=num_decodes * query_len,
max_seq_len=max(seq_lens),
causal=causal,
decode=SimpleNamespace(
# arange page ids so each request's window is an identifiable slice.
block_table=torch.arange(num_decodes * 64, dtype=torch.int32).reshape(
num_decodes, 64
),
seq_lens=torch.tensor(seq_lens, dtype=torch.int32),
),
)
def _run_forward_mqa(
seq_lens: list[int],
query_len: int,
causal: bool,
q_rows=None,
dcp_world_size: int = 1,
):
"""Drive forward_mqa with the decode kernel spied out.
Returns the ``(block_table, seq_lens)`` handed to ``decode_attention_fwd``.
"""
captured: dict = {}
def spy(q, kv_cache, kv_c_cache, o, lse, block_table, b_seq_len, *args, **kwargs):
captured["block_table"] = block_table.detach().clone()
captured["seq_lens"] = b_seq_len.detach().clone()
metadata = _make_metadata(seq_lens, query_len, causal)
rows = metadata.num_decode_tokens if q_rows is None else q_rows
q = torch.zeros(rows, NUM_HEADS, HEAD_SIZE, dtype=torch.bfloat16)
kv_cache = torch.zeros(
len(seq_lens) * 64, PAGE_SIZE, HEAD_SIZE, dtype=torch.bfloat16
)
layer = SimpleNamespace(_k_scale=torch.ones(1), layer_name="test")
with patch("vllm.v1.attention.backends.mla.triton_mla.decode_attention_fwd", spy):
_make_impl(dcp_world_size).forward_mqa(q, kv_cache, metadata, layer)
assert captured, "forward_mqa did not reach the decode kernel"
return captured
def test_multi_token_decode_flags_are_declared_together():
"""Capture probes max_query_len > 1, which only clears the assert in
build_for_cudagraph_capture once query_len_support raised the threshold."""
assert (
TritonMLAMetadataBuilder._cudagraph_support == AttentionCGSupport.UNIFORM_BATCH
)
assert TritonMLAMetadataBuilder.query_len_support == QueryLenSupport.UNIFORM
@pytest.mark.parametrize("query_len", [2, 3, 4, 5, 8])
def test_causal_block_rows_see_prefix_plus_own_position(query_len):
"""Causal verify: row t sees seq_len - (query_len - 1) + t KV entries.
Regression guard: the causal branch used to be skipped entirely, so a verify
token could attend the draft siblings it is supposed to be checking. Fails
on unmodified upstream, where no flatten happens at all for causal blocks.
"""
seq_lens = _seq_lens(query_len)
captured = _run_forward_mqa(seq_lens, query_len, causal=True)
got = captured["seq_lens"].tolist()
want = [s - (query_len - 1) + t for s in seq_lens for t in range(query_len)]
assert got == want, (
"causal flatten is not causal: verify row r*query_len+t must get "
f"seq_len_r - {query_len - 1} + t entries.\n"
f" seq_lens {seq_lens}\n got {got}\n expected {want}"
)
# Per-request invariants the arithmetic must hold regardless of query_len.
for r, seq_len in enumerate(seq_lens):
rows = got[r * query_len : (r + 1) * query_len]
assert rows == sorted(rows) and len(set(rows)) == query_len, (
f"request {r} rows must strictly increase, got {rows}"
)
assert rows[-1] == seq_len, "last row must see the full sequence"
assert rows[0] == seq_len - (query_len - 1)
assert all(n > 0 for n in rows), f"non-positive KV extent in {rows}"
def test_cudagraph_padding_rows_present_no_kv_extent():
"""Causal flattening pins cudagraph padding rows (seq_len 0) to exactly
zero KV extent, keeping them in the kernel's skipped branch."""
query_len = 4
seq_lens = _seq_lens(query_len) + [0]
captured = _run_forward_mqa(seq_lens, query_len, causal=True)
padding = captured["seq_lens"].tolist()[-query_len:]
assert padding == [0] * query_len, (
f"padding rows must present exactly zero KV extent, got {padding}"
)
@pytest.mark.parametrize("query_len", [2, 4])
def test_non_causal_block_rows_all_see_the_full_prefix(query_len):
"""The DSpark draft block is generated in one pass, so no row is masked."""
seq_lens = _seq_lens(query_len)
captured = _run_forward_mqa(seq_lens, query_len, causal=False)
want = [s for s in seq_lens for _ in range(query_len)]
assert captured["seq_lens"].tolist() == want
@pytest.mark.parametrize("causal", [True, False])
def test_block_table_is_expanded_to_match_q_rows(causal):
"""The kernel indexes block_table by program_id, so it needs one row per
query token -- a short block_table is read out of bounds."""
query_len = 4
captured = _run_forward_mqa(_seq_lens(query_len), query_len, causal=causal)
num_rows = len(CONTEXT_LENS) * query_len
assert captured["block_table"].shape[0] == num_rows
assert captured["seq_lens"].shape[0] == num_rows
# repeat_interleave, not repeat: a request's rows must stay adjacent.
for r in range(len(CONTEXT_LENS)):
block = captured["block_table"][r * query_len : (r + 1) * query_len]
assert torch.equal(block, block[:1].expand_as(block))
@pytest.mark.parametrize("causal", [True, False])
def test_single_token_decode_is_untouched(causal):
"""query_len == 1 must short-circuit: ordinary decode sees no expansion."""
seq_lens = _seq_lens(1)
captured = _run_forward_mqa(seq_lens, query_len=1, causal=causal)
assert captured["seq_lens"].tolist() == seq_lens
assert captured["block_table"].shape[0] == len(seq_lens)
def test_expansion_factor_follows_the_q_row_count():
"""The kernel indexes block_table and seq_lens by program_id, so the
expansion must follow q rows, not the token/request counts."""
num_rows = len(CONTEXT_LENS) * 4
captured = _run_forward_mqa(_seq_lens(1), query_len=1, causal=True, q_rows=num_rows)
assert captured["block_table"].shape[0] == num_rows
assert captured["seq_lens"].shape[0] == num_rows
def test_non_uniform_block_is_rejected():
"""Q rows must divide evenly across the decode requests."""
with pytest.raises(AssertionError, match="non-uniform decode block"):
_run_forward_mqa(_seq_lens(1), query_len=1, causal=True, q_rows=6)
def test_causal_multi_token_decode_is_rejected_under_dcp():
"""Per-row extents offset the global seq_len, but DCP passes the rank-local
slice, so the causal arithmetic would silently address the wrong KV."""
with pytest.raises(AssertionError, match="not supported with DCP"):
_run_forward_mqa(_seq_lens(4), query_len=4, causal=True, dcp_world_size=2)
# Non-causal rows all take the same extent, which stays correct when local.
_run_forward_mqa(_seq_lens(4), query_len=4, causal=False, dcp_world_size=2)
@@ -1,420 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Standalone unit tests for trtllm_prefill_attn_kvfp8_dequant.
Tests both contiguous and non-contiguous (cross-layer unified) KV cache
layouts against a pure-PyTorch reference implementation.
"""
import pytest
import torch
from vllm.platforms import current_platform
from vllm.v1.kv_cache_interface import KVCacheLayout
if current_platform.is_rocm():
pytest.skip(
"trtllm kvfp8 dequant is not supported on ROCm.",
allow_module_level=True,
)
FP8_DTYPE = current_platform.fp8_dtype()
NUM_BLOCKS = 128
def to_float8(x, dtype=None):
if dtype is None:
dtype = FP8_DTYPE
finfo = torch.finfo(dtype)
min_val, max_val = x.aminmax()
amax = torch.maximum(min_val.abs(), max_val.abs()).clamp(min=1e-12)
scale = finfo.max / amax * 0.1
x_scl_sat = (x * scale).clamp(min=finfo.min, max=finfo.max)
return x_scl_sat.to(dtype), scale.float().reciprocal()
def make_random_kv_cache(
num_blocks,
num_kv_heads,
block_size,
head_size,
layout=KVCacheLayout.LBHNC,
num_layers=None,
):
"""Create a random fp8 KV cache mimicking the production allocation.
Physical layout: [L, B, H, N, 2 * hs] permuted by ``layout.stride_order``
Returned view: (num_blocks, 2, num_kv_heads, block_size, head_size)
with non-contiguous strides under cross-layer layouts (they skip over
num_layers), matching the actual forward path.
"""
logical_4d = (num_blocks, num_kv_heads, block_size, 2 * head_size)
if num_layers is None:
num_layers = 1 if layout.is_layer_compact else 2
logical_5d = (num_layers, *logical_4d)
physical_5d = tuple(logical_5d[i] for i in layout.stride_order)
inv_order = [layout.stride_order.index(i) for i in range(5)]
raw_phys = torch.randn(*physical_5d, dtype=torch.bfloat16, device="cuda")
fp8_phys, scale = to_float8(raw_phys)
fp8_4d = fp8_phys.permute(*inv_order)[0]
kv_5d = fp8_4d.view(
num_blocks,
num_kv_heads,
block_size,
2,
head_size,
).permute(0, 3, 1, 2, 4)
return kv_5d, scale
def ref_dequant(kv_cache, block_tables, k_scale, v_scale, dequant_dtype):
"""Pure PyTorch reference: gather pages and dequantize fp8 -> dequant_dtype."""
batch_size, num_pages_per_seq = block_tables.shape
s = kv_cache.shape
out = torch.zeros(
batch_size * num_pages_per_seq + 1,
s[1],
s[2],
s[3],
s[4],
dtype=dequant_dtype,
device=kv_cache.device,
)
for b in range(batch_size):
for p in range(num_pages_per_seq):
page_idx = block_tables[b, p].item()
if page_idx <= 0:
continue
mock_idx = b * num_pages_per_seq + p + 1
out[mock_idx, 0] = (kv_cache[page_idx, 0].float() * k_scale.item()).to(
dequant_dtype
)
out[mock_idx, 1] = (kv_cache[page_idx, 1].float() * v_scale.item()).to(
dequant_dtype
)
return out
@pytest.mark.parametrize("num_kv_heads", [1, 8])
@pytest.mark.parametrize("head_size", [64, 128])
@pytest.mark.parametrize("block_size", [16, 32])
@pytest.mark.parametrize("batch_size", [1, 4])
@pytest.mark.parametrize("num_pages_per_seq", [3, 8])
@pytest.mark.parametrize("layout", list(KVCacheLayout))
@torch.inference_mode()
def test_trtllm_kvfp8_dequant(
num_kv_heads: int,
head_size: int,
block_size: int,
batch_size: int,
num_pages_per_seq: int,
layout: KVCacheLayout,
):
from vllm.v1.attention.backends.flashinfer import (
trtllm_prefill_attn_kvfp8_dequant,
)
torch.set_default_device("cuda")
kv_cache, scale = make_random_kv_cache(
NUM_BLOCKS,
num_kv_heads,
block_size,
head_size,
layout=layout,
)
k_scale = scale.clone()
v_scale = scale.clone()
block_tables = torch.randint(
1,
NUM_BLOCKS,
(batch_size, num_pages_per_seq),
dtype=torch.int32,
)
mock_kv_cache, mock_block_table = trtllm_prefill_attn_kvfp8_dequant(
kv_cache,
block_tables,
k_scale,
v_scale,
torch.bfloat16,
)
ref = ref_dequant(kv_cache, block_tables, k_scale, v_scale, torch.bfloat16)
expected_bt = torch.arange(
1,
batch_size * num_pages_per_seq + 1,
dtype=torch.int32,
device="cuda",
).reshape(batch_size, num_pages_per_seq)
torch.testing.assert_close(mock_block_table, expected_bt)
# Page 0 is padding (never written), compare only pages 1+
torch.testing.assert_close(mock_kv_cache[1:], ref[1:], atol=1e-3, rtol=1e-3)
@torch.inference_mode()
def test_block_tables_with_zero_pages():
"""Pages with index <= 0 must be skipped (early return in kernel)."""
from vllm.v1.attention.backends.flashinfer import (
trtllm_prefill_attn_kvfp8_dequant,
)
torch.set_default_device("cuda")
num_kv_heads, block_size, head_size = 8, 16, 64
kv_cache, scale = make_random_kv_cache(
NUM_BLOCKS,
num_kv_heads,
block_size,
head_size,
)
k_scale = v_scale = scale.clone()
# Mix of valid pages and zeros (padding)
block_tables = torch.tensor(
[[5, 0, 10], [0, 0, 0], [3, 7, 0]],
dtype=torch.int32,
device="cuda",
)
mock_kv_cache, _ = trtllm_prefill_attn_kvfp8_dequant(
kv_cache,
block_tables,
k_scale,
v_scale,
torch.bfloat16,
)
ref = ref_dequant(kv_cache, block_tables, k_scale, v_scale, torch.bfloat16)
# Only compare pages that were actually written (non-zero page indices)
for b in range(block_tables.shape[0]):
for p in range(block_tables.shape[1]):
if block_tables[b, p].item() > 0:
idx = b * block_tables.shape[1] + p + 1
torch.testing.assert_close(
mock_kv_cache[idx],
ref[idx],
atol=1e-3,
rtol=1e-3,
)
@torch.inference_mode()
def test_all_zero_block_tables():
"""All-zero block_tables: kernel should write nothing."""
from vllm.v1.attention.backends.flashinfer import (
trtllm_prefill_attn_kvfp8_dequant,
)
torch.set_default_device("cuda")
num_kv_heads, block_size, head_size = 4, 16, 64
kv_cache, scale = make_random_kv_cache(
NUM_BLOCKS,
num_kv_heads,
block_size,
head_size,
)
k_scale = v_scale = scale.clone()
block_tables = torch.zeros(2, 4, dtype=torch.int32, device="cuda")
# Should not crash even though no pages are valid
mock_kv_cache, mock_block_table = trtllm_prefill_attn_kvfp8_dequant(
kv_cache,
block_tables,
k_scale,
v_scale,
torch.bfloat16,
)
assert mock_kv_cache.shape[0] == 2 * 4 + 1
assert mock_block_table.shape == (2, 4)
@torch.inference_mode()
def test_different_k_v_scales():
"""Verify K and V are dequantized with independent scales."""
from vllm.v1.attention.backends.flashinfer import (
trtllm_prefill_attn_kvfp8_dequant,
)
torch.set_default_device("cuda")
num_kv_heads, block_size, head_size = 8, 16, 64
kv_cache, _ = make_random_kv_cache(
NUM_BLOCKS,
num_kv_heads,
block_size,
head_size,
)
k_scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
v_scale = torch.tensor([2.0], dtype=torch.float32, device="cuda")
block_tables = torch.tensor([[1, 2]], dtype=torch.int32, device="cuda")
mock_kv_cache, _ = trtllm_prefill_attn_kvfp8_dequant(
kv_cache,
block_tables,
k_scale,
v_scale,
torch.bfloat16,
)
ref = ref_dequant(kv_cache, block_tables, k_scale, v_scale, torch.bfloat16)
torch.testing.assert_close(mock_kv_cache[1:], ref[1:], atol=1e-3, rtol=1e-3)
@torch.inference_mode()
def test_single_page_per_seq():
"""Minimum grid dim 1 = 1 page per sequence."""
from vllm.v1.attention.backends.flashinfer import (
trtllm_prefill_attn_kvfp8_dequant,
)
torch.set_default_device("cuda")
num_kv_heads, block_size, head_size = 8, 16, 128
kv_cache, scale = make_random_kv_cache(
NUM_BLOCKS,
num_kv_heads,
block_size,
head_size,
)
k_scale = v_scale = scale.clone()
block_tables = torch.tensor([[5], [10], [20]], dtype=torch.int32, device="cuda")
mock_kv_cache, _ = trtllm_prefill_attn_kvfp8_dequant(
kv_cache,
block_tables,
k_scale,
v_scale,
torch.bfloat16,
)
ref = ref_dequant(kv_cache, block_tables, k_scale, v_scale, torch.bfloat16)
torch.testing.assert_close(mock_kv_cache[1:], ref[1:], atol=1e-3, rtol=1e-3)
@torch.inference_mode()
def test_large_page_indices():
"""Page indices near the top of the buffer stress offset arithmetic."""
from vllm.v1.attention.backends.flashinfer import (
trtllm_prefill_attn_kvfp8_dequant,
)
torch.set_default_device("cuda")
num_kv_heads, block_size, head_size = 8, 16, 128
large_num_blocks = 32768
kv_cache, scale = make_random_kv_cache(
large_num_blocks,
num_kv_heads,
block_size,
head_size,
)
k_scale = v_scale = scale.clone()
# Use page indices near the top of the buffer
block_tables = torch.tensor(
[[large_num_blocks - 1, large_num_blocks - 2, 1]],
dtype=torch.int32,
device="cuda",
)
mock_kv_cache, _ = trtllm_prefill_attn_kvfp8_dequant(
kv_cache,
block_tables,
k_scale,
v_scale,
torch.bfloat16,
)
ref = ref_dequant(kv_cache, block_tables, k_scale, v_scale, torch.bfloat16)
torch.testing.assert_close(mock_kv_cache[1:], ref[1:], atol=1e-3, rtol=1e-3)
@torch.inference_mode()
def test_large_block_size():
"""block_size=64 -> HEAD_STRIDE=8192, large tl.arange per thread block."""
from vllm.v1.attention.backends.flashinfer import (
trtllm_prefill_attn_kvfp8_dequant,
)
torch.set_default_device("cuda")
num_kv_heads, block_size, head_size = 4, 64, 128
kv_cache, scale = make_random_kv_cache(
NUM_BLOCKS,
num_kv_heads,
block_size,
head_size,
)
k_scale = v_scale = scale.clone()
block_tables = torch.randint(
1,
NUM_BLOCKS,
(2, 4),
dtype=torch.int32,
device="cuda",
)
mock_kv_cache, _ = trtllm_prefill_attn_kvfp8_dequant(
kv_cache,
block_tables,
k_scale,
v_scale,
torch.bfloat16,
)
ref = ref_dequant(kv_cache, block_tables, k_scale, v_scale, torch.bfloat16)
torch.testing.assert_close(mock_kv_cache[1:], ref[1:], atol=1e-3, rtol=1e-3)
@torch.inference_mode()
def test_cross_layer_many_layers():
"""Non-contiguous with 36 layers -- matches real gpt-oss-120b.
Cross-layer (BLHNC) strides are far from contiguous (factor of 36 in the gaps).
"""
from vllm.v1.attention.backends.flashinfer import (
trtllm_prefill_attn_kvfp8_dequant,
)
torch.set_default_device("cuda")
num_kv_heads, block_size, head_size = 8, 16, 64
kv_cache, scale = make_random_kv_cache(
NUM_BLOCKS,
num_kv_heads,
block_size,
head_size,
layout=KVCacheLayout.BLHNC,
num_layers=36,
)
k_scale = v_scale = scale.clone()
block_tables = torch.randint(
1,
NUM_BLOCKS,
(4, 6),
dtype=torch.int32,
device="cuda",
)
mock_kv_cache, _ = trtllm_prefill_attn_kvfp8_dequant(
kv_cache,
block_tables,
k_scale,
v_scale,
torch.bfloat16,
)
ref = ref_dequant(kv_cache, block_tables, k_scale, v_scale, torch.bfloat16)
torch.testing.assert_close(mock_kv_cache[1:], ref[1:], atol=1e-3, rtol=1e-3)
@@ -1,75 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the replicated-embedding fused gather/norm kernels
(``vllm.model_executor.layers.fused_embed_norm``).
The guarantee: enabling ``VLLM_REPLICATE_EMBED`` (replicated table + fused
kernels) must not change model outputs. The gathered residual is bit-exact and
the fused norms match the unfused reference.
"""
import pytest
import torch
from vllm.model_executor.layers.fused_embed_norm import (
fused_embed_eh_norm,
fused_embed_norm,
)
# The model-local (untouched) eh-norm the replicate path must match.
from vllm.models.deepseek_v32.common.kernels import fused_eh_norm
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
DTYPE = torch.bfloat16
VOCAB, HIDDEN, NUM_TOKENS, EPS = 8192, 4096, 129, 1e-6
requires_cuda = pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="fused embed/norm Triton kernels require a CUDA/ROCm device",
)
def _rmsnorm(x: torch.Tensor, w: torch.Tensor, eps: float) -> torch.Tensor:
# Full-precision (fp32) reference RMSNorm.
var = x.float().pow(2).mean(dim=-1, keepdim=True)
return x.float() * torch.rsqrt(var + eps) * w.float()
@requires_cuda
@torch.inference_mode()
def test_fused_embed_norm_matches_reference():
"""Main-model fusion: the residual is the exact gather and the second output
is a correct RMSNorm. The norm matches a full-precision reference to ~2 bf16
ulp (rtol 1e-2) -- that gap is bf16 rounding, not the kernel."""
set_random_seed(13)
table = torch.randn(VOCAB, HIDDEN, dtype=DTYPE, device="cuda")
ids = torch.randint(0, VOCAB, (NUM_TOKENS,), dtype=torch.int32, device="cuda")
weight = torch.empty(HIDDEN, dtype=DTYPE, device="cuda").normal_(1.0, 0.1)
residual, normed = fused_embed_norm(ids, table, chain_weight=weight, eps=EPS)
embeds = table[ids.long()]
torch.testing.assert_close(residual, embeds, atol=0.0, rtol=0.0)
torch.testing.assert_close(
normed.float(), _rmsnorm(embeds, weight, EPS), atol=1e-3, rtol=1e-2
)
@requires_cuda
@torch.inference_mode()
def test_fused_embed_eh_norm_matches_reference():
"""MTP fusion (folded gather) is bit-exact vs gathering the embeds and
feeding the untouched model-local ``fused_eh_norm``."""
set_random_seed(13)
table = torch.randn(VOCAB, HIDDEN, dtype=DTYPE, device="cuda")
ids = torch.randint(0, VOCAB, (NUM_TOKENS,), dtype=torch.int32, device="cuda")
prev = torch.randn(NUM_TOKENS, HIDDEN, dtype=DTYPE, device="cuda")
enorm_w = torch.randn(HIDDEN, dtype=DTYPE, device="cuda")
hnorm_w = torch.randn(HIDDEN, dtype=DTYPE, device="cuda")
positions = torch.arange(NUM_TOKENS, device="cuda") # includes pos 0
fused = fused_embed_eh_norm(positions, ids, table, prev, enorm_w, hnorm_w, EPS)
ref = fused_eh_norm(positions, table[ids.long()], prev, enorm_w, hnorm_w, EPS)
torch.testing.assert_close(fused, ref, atol=0.0, rtol=0.0)
@@ -1,303 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness + large-token-count launch tests for fused_q_kv_rmsnorm.
Before the grid-dim fix the kernel used grid ``(2, num_tokens)``, which hit
CUDA's 65535 grid-y cap for ``num_tokens >= 65536`` and failed with
``Triton Error [CUDA]: invalid argument`` at every large chunked-prefill
profile run. These tests pin the new grid layout.
"""
from __future__ import annotations
import pytest
import torch
from vllm.models.common.ops.fused_qk_rmsnorm import (
fused_q_kv_rmsnorm,
)
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="fused_q_kv_rmsnorm requires a CUDA/ROCm device",
)
def _ref_rmsnorm(x: torch.Tensor, w: torch.Tensor, eps: float) -> torch.Tensor:
x_f32 = x.to(torch.float32)
variance = x_f32.pow(2).mean(dim=-1, keepdim=True)
y = x_f32 * torch.rsqrt(variance + eps) * w.to(torch.float32)
return y.to(x.dtype)
@pytest.mark.parametrize("num_tokens", [1, 17, 1024, 8192])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
def test_fused_q_kv_rmsnorm_correctness(num_tokens: int, dtype: torch.dtype):
torch.manual_seed(0)
device = "cuda"
q_size, kv_size = 192, 576
qr = torch.randn(num_tokens, q_size, dtype=dtype, device=device)
kv = torch.randn(num_tokens, kv_size, dtype=dtype, device=device)
qw = torch.randn(q_size, dtype=dtype, device=device)
kvw = torch.randn(kv_size, dtype=dtype, device=device)
eps = 1e-6
qr_out, kv_out = fused_q_kv_rmsnorm(qr, kv, qw, kvw, eps)
qr_ref = _ref_rmsnorm(qr, qw, eps)
kv_ref = _ref_rmsnorm(kv, kvw, eps)
tol = dict(rtol=1e-2, atol=1e-2)
torch.testing.assert_close(qr_out, qr_ref, **tol)
torch.testing.assert_close(kv_out, kv_ref, **tol)
@pytest.mark.parametrize("num_tokens", [1, 16])
def test_fused_q_kv_rmsnorm_outputs_are_packed(num_tokens: int):
"""Regression guard: the outputs must be packed row-major even when the
inputs are column slices of a wider fused-projection buffer.
empty_like preserves the strides of size-1 dims, so with num_tokens == 1
a [1, q_size] slice of a [1, q_size + kv_size] buffer used to produce a
qr_out with row stride q_size + kv_size. Downstream dispatchers that
require packed row-major inputs then reject the tensor and silently fall
back to a slower GEMM path on every decode step."""
device = "cuda"
dtype = torch.bfloat16
q_size, kv_size = 192, 576
fused = torch.randn(num_tokens, q_size + kv_size, dtype=dtype, device=device)
qr, kv = fused.split([q_size, kv_size], dim=-1)
qw = torch.randn(q_size, dtype=dtype, device=device)
kvw = torch.randn(kv_size, dtype=dtype, device=device)
eps = 1e-6
qr_out, kv_out = fused_q_kv_rmsnorm(qr, kv, qw, kvw, eps)
assert qr_out.stride() == (q_size, 1)
assert kv_out.stride() == (kv_size, 1)
qr_ref = _ref_rmsnorm(qr, qw, eps)
kv_ref = _ref_rmsnorm(kv, kvw, eps)
tol = dict(rtol=1e-2, atol=1e-2)
torch.testing.assert_close(qr_out, qr_ref, **tol)
torch.testing.assert_close(kv_out, kv_ref, **tol)
@pytest.mark.parametrize("num_tokens", [65535, 65536, 131072])
def test_fused_q_kv_rmsnorm_launches_past_grid_y_cap(num_tokens: int):
"""Regression guard: grid used to be (2, num_tokens), hitting CUDA's
65535 grid-y cap at num_tokens >= 65536. The new grid (num_tokens, 2)
lifts that bound to 2**31-1."""
device = "cuda"
dtype = torch.bfloat16
q_size, kv_size = 192, 576
qr = torch.randn(num_tokens, q_size, dtype=dtype, device=device)
kv = torch.randn(num_tokens, kv_size, dtype=dtype, device=device)
qw = torch.randn(q_size, dtype=dtype, device=device)
kvw = torch.randn(kv_size, dtype=dtype, device=device)
qr_out, kv_out = fused_q_kv_rmsnorm(qr, kv, qw, kvw, 1e-6)
# spot-check a couple of rows against the torch reference
for row in (0, num_tokens // 2, num_tokens - 1):
torch.testing.assert_close(
qr_out[row],
_ref_rmsnorm(qr[row : row + 1], qw, 1e-6)[0],
rtol=1e-2,
atol=1e-2,
)
torch.testing.assert_close(
kv_out[row],
_ref_rmsnorm(kv[row : row + 1], kvw, 1e-6)[0],
rtol=1e-2,
atol=1e-2,
)
@pytest.mark.skipif(
not current_platform.has_device_capability(100), reason="MXFP8 needs Blackwell"
)
@pytest.mark.parametrize("num_tokens", [0, 1, 17, 128, 129, 1024])
@pytest.mark.parametrize("q_size", [384, 1280, 1344])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
def test_fused_q_kv_rmsnorm_quant_matches_separate(num_tokens, q_size, dtype):
"""Preserve native MXFP8 rounding and all swizzled scale padding bytes."""
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
mxfp8_e4m3_quantize,
)
from vllm.models.deepseek_v41.common.ops.query_quant import (
fused_q_kv_rmsnorm_quant,
)
torch.manual_seed(0)
kv_size = 512
x = torch.randn(num_tokens, q_size + kv_size, device="cuda", dtype=dtype)
x[:1] = 0
x[1:2] *= 1e-30
qr, kv = x.split([q_size, kv_size], -1)
qw = torch.randn(q_size, device="cuda", dtype=x.dtype)
kvw = torch.randn(kv_size, device="cuda", dtype=x.dtype)
eps = 1e-20
result = fused_q_kv_rmsnorm_quant(qr, kv, qw, kvw, eps)
assert result[0].orig_dtype == dtype
assert result[0].orig_shape == qr.shape
if not num_tokens:
assert result[0].data.shape == qr.shape
assert result[0].scale.numel() == 0
assert result[1].shape == kv.shape
return
qr_ref, kv_ref = fused_q_kv_rmsnorm(qr, kv, qw, kvw, eps)
q_ref, scale_ref = mxfp8_e4m3_quantize(qr_ref, is_sf_swizzled_layout=True)
for replay in (False, True):
if replay:
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
result = fused_q_kv_rmsnorm_quant(qr, kv, qw, kvw, eps)
graph.replay()
torch.testing.assert_close(
result[0].data.view(torch.uint8), q_ref.view(torch.uint8), rtol=0, atol=0
)
torch.testing.assert_close(result[0].scale, scale_ref, rtol=0, atol=0)
torch.testing.assert_close(result[1], kv_ref, rtol=0, atol=0)
@pytest.mark.skipif(
not current_platform.has_device_capability(100), reason="MXFP8 needs Blackwell"
)
@pytest.mark.parametrize("backend", ["cute-dsl", "cutlass"])
@pytest.mark.parametrize("num_tokens", [1, 17, 256])
@pytest.mark.parametrize("projection", ["attention", "indexer"])
@pytest.mark.parametrize("bias", [False, True])
def test_shared_query_quant_preserves_projection(
num_tokens, backend, projection, bias, monkeypatch
):
"""The native-checkpoint projection keeps identical GEMM outputs."""
from dataclasses import replace
from types import SimpleNamespace
from typing import Any, cast
from vllm.config import (
CompilationConfig,
CompilationMode,
VllmConfig,
set_current_vllm_config,
)
from vllm.model_executor.kernels.linear.mxfp8 import Mxfp8LinearLayerConfig
from vllm.model_executor.kernels.linear.mxfp8.flashinfer import (
FlashInferCutedslMxfp8LinearKernel,
FlashInferCutlassMxfp8LinearKernel,
)
from vllm.model_executor.layers.fusion.quant_activation import (
expose_input_quant_key,
)
from vllm.model_executor.layers.linear import ColumnParallelLinear, ReplicatedLinear
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptLinearMethod,
)
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
mxfp8_e4m3_quantize,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import kNvfp4Dynamic
from vllm.models.deepseek_v41.attention import (
DeepseekV4Attention,
DeepseekV4Indexer,
)
from vllm.models.deepseek_v41.common.ops.query_quant import (
can_fuse_query_quant,
fused_q_kv_rmsnorm_quant,
)
from vllm.models.deepseek_v41.quant_config import DeepseekV4FP8Config
cls: (
type[FlashInferCutedslMxfp8LinearKernel]
| type[FlashInferCutlassMxfp8LinearKernel]
)
if backend == "cute-dsl":
cls = FlashInferCutedslMxfp8LinearKernel
else:
cls = FlashInferCutlassMxfp8LinearKernel
supported, reason = cls.is_supported()
if not supported:
pytest.skip(reason)
torch.manual_seed(0)
with set_current_vllm_config(
VllmConfig(compilation_config=CompilationConfig(mode=CompilationMode.NONE))
):
# Match the checkpoint's TP4 attention and replicated indexer widths.
if projection == "attention":
linear = ColumnParallelLinear.__new__(ColumnParallelLinear)
output_size = 8192
else:
linear = ReplicatedLinear.__new__(ReplicatedLinear)
output_size = 4096
torch.nn.Module.__init__(linear)
linear.bias = (
torch.nn.Parameter(
torch.randn(output_size, device="cuda", dtype=torch.bfloat16)
)
if bias
else None
)
linear.skip_bias_add = False
linear.return_bias = projection == "indexer"
linear.gather_output = False
w, ws = mxfp8_e4m3_quantize(
torch.randn(output_size, 1280, device="cuda", dtype=torch.bfloat16)
)
linear.weight = torch.nn.Parameter(w, requires_grad=False)
linear.weight_scale = torch.nn.Parameter(ws, requires_grad=False)
kernel = cls(Mxfp8LinearLayerConfig(weight_shape=w.shape))
quant_config = DeepseekV4FP8Config.from_config(
{
"quant_method": "fp8",
"activation_scheme": "dynamic",
"weight_block_size": [32, 32],
"scale_fmt": "ue8m0",
"expert_dtype": "fp4",
}
)
method = quant_config.get_quant_method(linear, "model.layers.2.attn.wq_b")
assert type(method) is ModelOptLinearMethod
method.out_dtype = torch.bfloat16
method.kernel = kernel
linear.quant_method = method
kernel.process_weights_after_loading(linear)
expose_input_quant_key(linear, kernel)
assert can_fuse_query_quant([linear, linear])
x = torch.randn(num_tokens, 1792, device="cuda", dtype=torch.bfloat16)
qr, kv = x.split([1280, 512], -1)
qw = torch.randn(1280, device="cuda", dtype=x.dtype)
kvw = torch.randn(512, device="cuda", dtype=x.dtype)
qr_ref, _ = fused_q_kv_rmsnorm(qr, kv, qw, kvw, 1e-20)
q, _ = fused_q_kv_rmsnorm_quant(qr, kv, qw, kvw, 1e-20)
owner = cast(Any, SimpleNamespace(wq_b=linear))
project = (
DeepseekV4Attention._wq_b_proj
if projection == "attention"
else DeepseekV4Indexer._wq_b_proj
)
expected = project(owner, qr_ref)
def reject_requantization(*args, **kwargs):
raise AssertionError("pre-quantized input must skip quantization")
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mxfp8.flashinfer.mxfp8_e4m3_quantize",
reject_requantization,
)
actual = project(owner, q)
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
actual = project(owner, q)
graph.replay()
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
assert not can_fuse_query_quant(
[linear, torch.nn.Linear(1280, 256, bias=False)]
)
reshaped = project(
owner, replace(q, orig_shape=torch.Size((1, num_tokens, 1280)))
)
torch.testing.assert_close(reshaped, expected.unsqueeze(0), rtol=0, atol=0)
with pytest.raises(AssertionError, match="consumer kernel"):
linear(replace(q, quant_key=kNvfp4Dynamic))
@@ -1,120 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Accuracy tests for the fused Triton bilinear position-embedding kernel.
Compares ``triton_pos_embed_interpolate`` against the pure-PyTorch
``pos_embed_interpolate_native`` across a variety of grid shapes and dtypes.
"""
import pytest
import torch
from vllm.triton_utils import HAS_TRITON
if HAS_TRITON:
from vllm.model_executor.models.qwen3_vl import (
pos_embed_interpolate_native,
triton_pos_embed_interpolate,
)
DTYPES = [torch.float32, torch.bfloat16]
# Qwen3-VL default
NUM_GRID_PER_SIDE = 48
SPATIAL_MERGE_SIZE = 2
HIDDEN_DIM = 1152
# 4 square + 4 non-square grids (h, w divisible by spatial_merge_size=2)
SQUARE_GRIDS = [(1, 4, 4), (1, 16, 16), (1, 32, 32), (1, 48, 48)]
NON_SQUARE_GRIDS = [(1, 8, 16), (1, 14, 20), (1, 32, 48), (1, 60, 80)]
ALL_GRIDS = SQUARE_GRIDS + NON_SQUARE_GRIDS
@pytest.mark.skipif(not HAS_TRITON, reason="Triton not available")
@pytest.mark.parametrize("dtype", DTYPES, ids=lambda d: str(d).split(".")[-1])
@pytest.mark.parametrize(
"grid_thw",
ALL_GRIDS,
ids=[f"{t}x{h}x{w}" for t, h, w in ALL_GRIDS],
)
def test_triton_matches_native(
grid_thw: tuple[int, int, int],
dtype: torch.dtype,
) -> None:
"""Triton kernel output must match the native PyTorch implementation."""
t, h, w = grid_thw
device = "cuda"
# Scale to match real Qwen3-VL pos_embed weight distribution (std~0.23).
torch.manual_seed(42)
embed_weight = (
torch.randn(
NUM_GRID_PER_SIDE * NUM_GRID_PER_SIDE,
HIDDEN_DIM,
device=device,
dtype=dtype,
)
* 0.25
)
native_out = pos_embed_interpolate_native(
embed_weight, t, h, w, NUM_GRID_PER_SIDE, SPATIAL_MERGE_SIZE, dtype
)
triton_out = triton_pos_embed_interpolate(
embed_weight, t, h, w, NUM_GRID_PER_SIDE, SPATIAL_MERGE_SIZE, dtype
)
assert native_out.shape == triton_out.shape, (
f"Shape mismatch: native {native_out.shape} vs triton {triton_out.shape}"
)
# Small numerical differences arise from the precomputed h/w_scale
# in the triton kernel vs torch.linspace in the native path, which can
# cause single-ULP output differences
# in a handful of elements.
atol = {torch.float32: 5e-5, torch.bfloat16: 1e-2}[dtype]
rtol = {torch.float32: 1e-5, torch.bfloat16: 1e-2}[dtype]
torch.testing.assert_close(triton_out, native_out, atol=atol, rtol=rtol)
@pytest.mark.skipif(not HAS_TRITON, reason="Triton not available")
@pytest.mark.parametrize("dtype", DTYPES, ids=lambda d: str(d).split(".")[-1])
def test_temporal_repeat(dtype: torch.dtype) -> None:
"""Verify temporal dimension t > 1 correctly repeats the spatial pattern."""
device = "cuda"
h, w = 16, 16
t_single, t_multi = 1, 3
# Scale to match real Qwen3-VL pos_embed weight distribution (std~0.23).
torch.manual_seed(42)
embed_weight = (
torch.randn(
NUM_GRID_PER_SIDE * NUM_GRID_PER_SIDE,
HIDDEN_DIM,
device=device,
dtype=dtype,
)
* 0.25
)
out_single = triton_pos_embed_interpolate(
embed_weight,
t_single,
h,
w,
NUM_GRID_PER_SIDE,
SPATIAL_MERGE_SIZE,
dtype,
)
out_multi = triton_pos_embed_interpolate(
embed_weight,
t_multi,
h,
w,
NUM_GRID_PER_SIDE,
SPATIAL_MERGE_SIZE,
dtype,
)
expected = out_single.repeat(t_multi, 1)
torch.testing.assert_close(out_multi, expected, atol=0, rtol=0)
@@ -1,250 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the fused_qk_norm_rope helion kernel.
Run `pytest tests/kernels/helion/test_fused_qk_norm_rope.py`.
"""
from typing import Any
import pytest
import torch
from torch._subclasses.fake_tensor import FakeTensorMode
from tests.kernels.helion.utils import skip_if_platform_unsupported
from vllm.benchmarks.lib.utils import default_vllm_config
from vllm.kernels.helion.case_key import CaseKey
from vllm.kernels.helion.config_manager import ConfigManager
from vllm.kernels.helion.ops.fused_qk_norm_rope import (
_pick_cache,
baseline,
fused_qk_norm_rope,
pick_config,
)
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
from vllm.utils.import_utils import has_helion
if not has_helion():
pytest.skip(
"Helion is not installed. Install with: pip install vllm[helion]",
allow_module_level=True,
)
@default_vllm_config()
def _generate_fake_input(
num_tokens: int, num_q_heads: int, num_kv_heads: int
) -> tuple[Any, ...]:
with FakeTensorMode():
head_dim = 128
eps = 1e-6
is_neox = True
rotary_ratio = 1.0
device = "cuda"
dtype = torch.bfloat16
total_dim = (num_q_heads + 2 * num_kv_heads) * head_dim
qkv = torch.randn(num_tokens, total_dim, dtype=dtype, device=device)
positions = torch.arange(num_tokens, dtype=torch.long, device=device)
q_weight = torch.normal(
mean=1.0,
std=1.0,
size=(head_dim,),
dtype=qkv.dtype,
device=device,
)
k_weight = torch.normal(
mean=1.0,
std=1.0,
size=(head_dim,),
dtype=qkv.dtype,
device=device,
)
rotary_dim = int(head_dim * rotary_ratio)
rope = RotaryEmbedding(
head_size=head_dim,
rotary_dim=rotary_dim,
max_position_embeddings=4096,
base=10000.0,
is_neox_style=is_neox,
dtype=dtype,
).to(device)
args = (
qkv,
num_q_heads,
num_kv_heads,
num_kv_heads,
head_dim,
eps,
q_weight,
k_weight,
rope.cos_sin_cache,
is_neox,
positions.view(-1),
)
return args
@pytest.fixture(autouse=True)
def reset_config_manager_singleton():
ConfigManager.reset_instance()
ConfigManager()
yield
ConfigManager.reset_instance()
class TestFusedQkNormRopeConfigPicker:
def setup_method(self):
_pick_cache.clear()
def test_config_picker_exact_match(self):
config_keys = [
CaseKey({"q_heads": 2048, "kv_heads": 64, "num_tokens": 16}),
CaseKey({"q_heads": 4096, "kv_heads": 128, "num_tokens": 16}),
]
args = _generate_fake_input(16, 4096, 128)
selected_key = pick_config(args, config_keys)
assert selected_key == CaseKey(
{"q_heads": 4096, "kv_heads": 128, "num_tokens": 16}
)
def test_config_picker_closest_match(self):
config_keys = [
CaseKey({"q_heads": 2048, "kv_heads": 64, "num_tokens": 16}),
CaseKey({"q_heads": 2048, "kv_heads": 64, "num_tokens": 32}),
CaseKey({"q_heads": 2048, "kv_heads": 128, "num_tokens": 16}),
CaseKey({"q_heads": 2048, "kv_heads": 128, "num_tokens": 32}),
CaseKey({"q_heads": 4096, "kv_heads": 64, "num_tokens": 16}),
CaseKey({"q_heads": 4096, "kv_heads": 64, "num_tokens": 32}),
CaseKey({"q_heads": 4096, "kv_heads": 128, "num_tokens": 16}),
CaseKey({"q_heads": 4096, "kv_heads": 128, "num_tokens": 32}),
]
args = _generate_fake_input(20, 3000, 70)
selected_key = pick_config(args, config_keys)
assert selected_key == CaseKey(
{"q_heads": 2048, "kv_heads": 64, "num_tokens": 32}
)
def test_config_picker_no_configs(self):
config_keys: list[CaseKey] = []
args = _generate_fake_input(16, 4096, 128)
selected_key = pick_config(args, config_keys)
assert selected_key is None
def test_config_picker_fallback_to_largest(self):
config_keys = [
CaseKey({"q_heads": 2048, "kv_heads": 64, "num_tokens": 16}),
CaseKey({"q_heads": 2048, "kv_heads": 64, "num_tokens": 32}),
CaseKey({"q_heads": 2048, "kv_heads": 128, "num_tokens": 16}),
CaseKey({"q_heads": 2048, "kv_heads": 128, "num_tokens": 32}),
CaseKey({"q_heads": 4096, "kv_heads": 64, "num_tokens": 16}),
CaseKey({"q_heads": 4096, "kv_heads": 64, "num_tokens": 32}),
CaseKey({"q_heads": 4096, "kv_heads": 128, "num_tokens": 16}),
CaseKey({"q_heads": 4096, "kv_heads": 128, "num_tokens": 32}),
]
args = _generate_fake_input(64, 8192, 256)
selected_key = pick_config(args, config_keys)
assert selected_key == CaseKey(
{"q_heads": 4096, "kv_heads": 128, "num_tokens": 32}
)
class TestFusedQkNormRopeCorrectness:
@pytest.mark.parametrize(
"num_heads, num_kv_heads, head_dim", [(16, 4, 128), (64, 8, 128)]
)
@pytest.mark.parametrize("num_tokens", [1, 7, 1024, 1025])
@pytest.mark.parametrize("is_neox", [False, True])
@pytest.mark.parametrize("rotary_ratio", [1.0, 0.5, 0.25])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@default_vllm_config()
def test_fused_qk_norm_rope(
self,
num_heads: int,
num_kv_heads: int,
head_dim: int,
num_tokens: int,
is_neox: bool,
rotary_ratio: float,
dtype: torch.dtype,
):
skip_if_platform_unsupported("fused_qk_norm_rope")
torch.manual_seed(42)
eps = 1e-6
device = "cuda"
total_dim = (num_heads + 2 * num_kv_heads) * head_dim
ref_qkv = torch.empty(
num_tokens, total_dim, dtype=dtype, device=device
).uniform_(-0.1, 0.1)
ops_qkv = ref_qkv.clone()
positions = torch.arange(num_tokens, dtype=torch.long, device=device)
q_weight = torch.empty(head_dim, dtype=dtype, device=device).uniform_(0.8, 1.2)
k_weight = torch.empty(head_dim, dtype=dtype, device=device).uniform_(0.8, 1.2)
rotary_dim = int(head_dim * rotary_ratio)
rope = RotaryEmbedding(
head_size=head_dim,
rotary_dim=rotary_dim,
max_position_embeddings=40960,
base=10000.0,
is_neox_style=is_neox,
dtype=dtype,
).to(device)
baseline(
ref_qkv,
num_heads,
num_kv_heads,
num_kv_heads,
head_dim,
eps,
q_weight,
k_weight,
rope.cos_sin_cache,
is_neox,
positions.view(-1),
)
fused_qk_norm_rope(
ops_qkv,
num_heads,
num_kv_heads,
num_kv_heads,
head_dim,
eps,
q_weight,
k_weight,
rope.cos_sin_cache,
is_neox,
positions.view(-1),
)
if dtype == torch.bfloat16:
atol = 5e-2
rtol = 5e-2
else:
atol = 1e-2
rtol = 1e-2
torch.testing.assert_close(
ref_qkv,
ops_qkv,
atol=atol,
rtol=rtol,
)
class TestFusedQkNormRopeIntegration:
def test_kernel_registration_integration(self):
from vllm.kernels.helion.register import get_registered_kernels
registered_kernels = get_registered_kernels()
assert "fused_qk_norm_rope" in registered_kernels
kernel_wrapper = registered_kernels["fused_qk_norm_rope"]
assert kernel_wrapper.op_name == "fused_qk_norm_rope"
assert kernel_wrapper._config_picker is not None
assert kernel_wrapper._mutates_args == ["qkv"]
@@ -1,302 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Integration test for the non-spec decode split in
``GatedDeltaNet._forward_core``.
On a pure non-spec batch that mixes prefills with 1-token decodes, the layer
peels the decodes (the contiguous decode-first front slice) off to
``fused_sigmoid_gating_delta_rule_update`` -- the same recurrent update kernel
the spec-decode path uses -- and runs only the prefill tail through
``chunk_gated_delta_rule``. This must produce the same core-attention output and
the same ssm-state pool update as running *everything* through
``chunk_gated_delta_rule`` (the previous behavior).
Both paths are exercised through the REAL ``_forward_core``:
* ``meta_split`` is built by the real ``GDNAttentionMetadataBuilder`` for a
mixed batch, so ``num_decodes > 0`` triggers the peel (and the builder rebases
``chunk_indices``/``chunk_offsets`` to the prefill-only tail).
* ``meta_unified`` is the same metadata with the decodes reclassified as
prefills and full-batch chunk metadata, which forces ``_forward_core`` through
the existing chunk-only path on identical inputs (the conv is unified over all
non-spec tokens in both paths, so it cancels out and only the recurrent split
is compared).
The Triton/FLA chunk backend is forced so the prefill-only ``chunk_indices``
must stay consistent with the rebased ``cu_seqlens`` (a stringent, backend
portable check of the split wiring).
"""
from __future__ import annotations
import dataclasses
import types
from unittest.mock import patch
import pytest
import torch
from vllm.platforms import current_platform
if not (
current_platform.is_cuda() and current_platform.is_device_capability_family(100)
):
pytest.skip(
reason="GDN _forward_core split test uses the CuteDSL prefill backend "
"(requires CUDA SM10x).",
allow_module_level=True,
)
from tests.v1.attention.utils import ( # noqa: E402
BatchSpec,
create_common_attn_metadata,
create_vllm_config,
)
from vllm.config import set_current_vllm_config # noqa: E402
from vllm.model_executor.layers.mamba.gdn import qwen_gdn_linear_attn # noqa: E402
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import ( # noqa: E402
ChunkGatedDeltaRule,
QwenGatedDeltaNetAttention,
)
from vllm.model_executor.layers.mamba.mamba_utils import ( # noqa: E402
MambaStateShapeCalculator,
)
from vllm.third_party.flash_linear_attention.ops.index import ( # noqa: E402
prepare_chunk_indices,
prepare_chunk_offsets,
)
from vllm.third_party.flash_linear_attention.ops.utils import ( # noqa: E402
FLA_CHUNK_SIZE,
)
from vllm.v1.attention.backends.gdn_attn import ( # noqa: E402
GDNAttentionMetadataBuilder,
)
from vllm.v1.kv_cache_interface import MambaSpec # noqa: E402
# Small GDN dims; head_k_dim/head_v_dim=128 keeps the chunk/update kernels happy.
H = 4 # num key heads
HV = 8 # num value heads
K = 128 # head_k_dim
V = 128 # head_v_dim
CONV_KERNEL = 4
KEY_DIM = H * K
VALUE_DIM = HV * V
CONV_DIM = 2 * KEY_DIM + VALUE_DIM
BLOCK_SIZE = 16
PREFIX = "model.layers.0.linear_attn"
def _make_vllm_config():
# A small, ungated GDN model whose config is cached locally; only the config
# (scheduler/cache/compilation/hf) is used here, never the weights. Inject
# linear_key_head_dim=128 and request the CuteDSL prefill backend -- the
# supported GDN chunk kernel on Blackwell (the Triton/FLA chunk kernel is
# unsupported on SM10x). CuteDSL consumes chunk_indices/chunk_offsets, so
# this also exercises the prefill-only chunk-metadata wiring.
cfg = create_vllm_config(
model_name="Qwen/Qwen3.5-0.8B",
block_size=BLOCK_SIZE,
hf_config_override={"linear_key_head_dim": K},
)
cfg.additional_config = {"gdn_prefill_backend": "cutedsl"}
return cfg
def _build_layer(
vllm_config, conv_state, ssm_state, A_log, dt_bias, conv_weight, conv_bias
):
"""A minimal object that runs the real ``_forward_core`` bound to it."""
layer = types.SimpleNamespace()
layer.prefix = PREFIX
layer.enable_packed_recurrent_decode = False
layer.tp_size = 1
layer.num_k_heads = H
layer.num_v_heads = HV
layer.head_k_dim = K
layer.head_v_dim = V
layer.key_dim = KEY_DIM
layer.value_dim = VALUE_DIM
layer.activation = "silu"
layer.A_log = A_log
layer.dt_bias = dt_bias
layer.conv1d = types.SimpleNamespace(weight=conv_weight, bias=conv_bias)
layer.kv_cache = (conv_state, ssm_state)
with set_current_vllm_config(vllm_config):
layer.chunk_gated_delta_rule = ChunkGatedDeltaRule()
for name in (
"rearrange_mixed_qkv",
"_forward_core",
):
setattr(
layer,
name,
types.MethodType(getattr(QwenGatedDeltaNetAttention, name), layer),
)
return layer
def _run_forward_core(layer, meta, mixed_qkv, b, a, num_tokens):
core_attn_out = torch.zeros(
num_tokens, HV, V, dtype=mixed_qkv.dtype, device=mixed_qkv.device
)
ctx = types.SimpleNamespace(attn_metadata={PREFIX: meta})
with patch.object(qwen_gdn_linear_attn, "get_forward_context", return_value=ctx):
layer._forward_core(
mixed_qkv=mixed_qkv.clone(),
b=b.clone(),
a=a.clone(),
core_attn_out=core_attn_out,
)
return core_attn_out
@pytest.mark.parametrize("state_dtype", [torch.bfloat16, torch.float32])
@pytest.mark.parametrize("num_decodes,prefill_lens", [(3, [512, 300]), (4, [64, 5])])
@pytest.mark.parametrize("fresh_prefill", [False, True])
def test_forward_core_split_matches_unified(
state_dtype: torch.dtype,
num_decodes: int,
prefill_lens: list[int],
fresh_prefill: bool,
) -> None:
torch.manual_seed(0)
device = torch.device("cuda")
vllm_config = _make_vllm_config()
# Decode-first batch: D 1-token decodes (with context), then the prefills.
decode_seq_lens = [64] * num_decodes
prefill_seq_lens = [
pl if (fresh_prefill and i == 0) else pl + 37
for i, pl in enumerate(prefill_lens)
]
seq_lens = decode_seq_lens + prefill_seq_lens
query_lens = [1] * num_decodes + list(prefill_lens)
batch = BatchSpec(seq_lens=seq_lens, query_lens=query_lens)
builder = GDNAttentionMetadataBuilder(
# GDNAttentionMetadataBuilder declares kv_cache_spec as AttentionSpec but
# asserts isinstance(kv_cache_spec, MambaSpec). MambaSpec is correct here.
kv_cache_spec=MambaSpec(
block_size=BLOCK_SIZE, shapes=((16, 64),), dtypes=(torch.float16,)
),
layer_names=[PREFIX],
vllm_config=vllm_config,
device=device,
)
common = create_common_attn_metadata(
batch, BLOCK_SIZE, device, arange_block_indices=True
)
with set_current_vllm_config(vllm_config):
meta_split = builder.build(common_prefix_len=0, common_attn_metadata=common)
assert meta_split.spec_sequence_masks is None
assert meta_split.num_decodes == num_decodes
assert meta_split.num_prefills == len(prefill_lens)
assert meta_split.num_decode_tokens == num_decodes
assert builder.gdn_prefill_backend == "cutedsl"
num_tokens = sum(query_lens)
# Full-batch chunk metadata for the unified reference path, built the same
# way the builder would for a non-split batch (backend-matched).
cu_full = meta_split.non_spec_query_start_loc
assert cu_full is not None
if builder.gdn_prefill_backend == "cutedsl":
from vllm.model_executor.layers.mamba.ops.gdn_chunk_cutedsl import (
prepare_metadata_cutedsl,
)
full_ci, full_co = prepare_metadata_cutedsl(
cu_full, int(cu_full[-1].item()), FLA_CHUNK_SIZE
)
else:
cu_full_cpu = cu_full.cpu()
full_ci = prepare_chunk_indices(cu_full_cpu, FLA_CHUNK_SIZE).to(device)
full_co = prepare_chunk_offsets(cu_full_cpu, FLA_CHUNK_SIZE).to(device)
meta_unified = dataclasses.replace(
meta_split,
num_decodes=0,
num_decode_tokens=0,
num_prefills=meta_split.num_decodes + meta_split.num_prefills,
num_prefill_tokens=(
meta_split.num_decode_tokens + meta_split.num_prefill_tokens
),
chunk_indices=full_ci,
chunk_offsets=full_co,
# Unified path: the chunk kernel processes the full non-spec batch.
prefill_query_start_loc=meta_split.non_spec_query_start_loc,
prefill_state_indices=meta_split.non_spec_state_indices_tensor,
prefill_has_initial_state=meta_split.has_initial_state,
)
# Size the state pools from the indices the builder actually produced.
assert meta_split.non_spec_state_indices_tensor is not None
pool_size = int(meta_split.non_spec_state_indices_tensor.max().item()) + 1
conv_state_shape, temporal_state_shape = (
MambaStateShapeCalculator.gated_delta_net_state_shape(
1, H, HV, K, V, CONV_KERNEL, num_spec=0
)
)
conv_state0 = (
torch.randn(pool_size, *conv_state_shape, dtype=torch.bfloat16, device=device)
* 0.05
)
ssm_state0 = (
torch.randn(pool_size, *temporal_state_shape, dtype=state_dtype, device=device)
* 0.05
)
A_log = torch.randn(HV, dtype=torch.float32, device=device) * 0.1
dt_bias = torch.randn(HV, dtype=torch.float32, device=device) * 0.1
conv_weight = (
torch.randn(CONV_DIM, 1, CONV_KERNEL, dtype=torch.bfloat16, device=device) * 0.1
)
conv_bias = torch.randn(CONV_DIM, dtype=torch.bfloat16, device=device) * 0.1
mixed_qkv = (
torch.randn(num_tokens, CONV_DIM, dtype=torch.bfloat16, device=device) * 0.1
)
a = torch.randn(num_tokens, HV, dtype=torch.bfloat16, device=device) * 0.1
b = torch.randn(num_tokens, HV, dtype=torch.bfloat16, device=device) * 0.1
# ---- Split path (real _forward_core, meta_split) ----
conv_state_split = conv_state0.clone()
ssm_state_split = ssm_state0.clone()
layer_split = _build_layer(
vllm_config,
conv_state_split,
ssm_state_split,
A_log,
dt_bias,
conv_weight,
conv_bias,
)
out_split = _run_forward_core(layer_split, meta_split, mixed_qkv, b, a, num_tokens)
# ---- Unified path (real _forward_core, meta_unified) ----
conv_state_unified = conv_state0.clone()
ssm_state_unified = ssm_state0.clone()
layer_unified = _build_layer(
vllm_config,
conv_state_unified,
ssm_state_unified,
A_log,
dt_bias,
conv_weight,
conv_bias,
)
out_unified = _run_forward_core(
layer_unified, meta_unified, mixed_qkv, b, a, num_tokens
)
# Conv is unified in both paths, so the conv-state update must be identical.
torch.testing.assert_close(conv_state_split, conv_state_unified, atol=0, rtol=0)
# Chunk vs. recurrent update accumulate in different orders; mirror the
# tolerances used by the kernel-level parity test.
if state_dtype == torch.float32:
atol = rtol = 2e-2
else:
atol = rtol = 6e-2
torch.testing.assert_close(out_split, out_unified, atol=atol, rtol=rtol)
torch.testing.assert_close(ssm_state_split, ssm_state_unified, atol=atol, rtol=rtol)
@@ -1,277 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
import pytest
import torch
import torch.nn.functional as F
from vllm.platforms import current_platform
if not (
current_platform.is_cuda() and current_platform.is_device_capability_family(100)
):
pytest.skip(
reason="GDN CuteDSL prefill requires CUDA SM10x.",
allow_module_level=True,
)
from vllm.model_executor.layers.mamba.ops.gdn_chunk_cutedsl import ( # noqa: E402
chunk_gated_delta_rule_cutedsl,
prepare_metadata_cutedsl,
)
from vllm.third_party.flash_linear_attention.ops import ( # noqa: E402
chunk_gated_delta_rule,
)
from vllm.third_party.flash_linear_attention.ops.index import ( # noqa: E402
prepare_chunk_indices,
prepare_chunk_offsets,
)
@pytest.mark.parametrize("num_seqs", [1, 5, 257])
@pytest.mark.parametrize("state_dtype", [torch.bfloat16, torch.float32])
def test_gdn_chunk_cutedsl_correctness(num_seqs: int, state_dtype: torch.dtype):
rng_cpu = torch.Generator("cpu").manual_seed(1234)
rng = torch.Generator("cuda").manual_seed(2345)
seq_lens = torch.randint(1, 130, (num_seqs,), dtype=torch.int32, generator=rng_cpu)
cu_seqlens = torch.zeros(num_seqs + 1, device="cuda", dtype=torch.int32)
cu_seqlens[1:] = seq_lens.to(device="cuda").cumsum(0)
total_tokens = int(cu_seqlens[-1].item())
num_k_heads = 4
num_v_heads = 8
head_k_dim = 128
head_v_dim = 128
dtype = torch.bfloat16
q = torch.randn(
1,
total_tokens,
num_k_heads,
head_k_dim,
device="cuda",
dtype=dtype,
generator=rng,
)
k = torch.randn_like(q, generator=rng)
v = torch.randn(
1,
total_tokens,
num_v_heads,
head_v_dim,
device="cuda",
dtype=dtype,
generator=rng,
)
q = F.normalize(q.float(), p=2, dim=-1).to(dtype)
k = F.normalize(k.float(), p=2, dim=-1).to(dtype)
a = torch.randn(
1, total_tokens, num_v_heads, device="cuda", dtype=dtype, generator=rng
)
b = torch.randn(
1, total_tokens, num_v_heads, device="cuda", dtype=dtype, generator=rng
)
# Match upstream FLA GatedDeltaNet synthetic initialization:
# https://github.com/fla-org/flash-linear-attention/blob/main/fla/layers/gated_deltanet.py
A = torch.empty(num_v_heads, device="cuda", dtype=torch.float32).uniform_(
0, 16, generator=rng
)
A_log = torch.log(A)
dt = torch.exp(
torch.rand(num_v_heads, device="cuda", dtype=torch.float32, generator=rng)
* (math.log(0.1) - math.log(0.001))
+ math.log(0.001)
)
dt = torch.clamp(dt, min=1e-4)
dt_bias = dt + torch.log(-torch.expm1(-dt))
g = -A_log.exp().view(1, 1, num_v_heads) * F.softplus(
a.float() + dt_bias.view(1, 1, num_v_heads)
)
beta = torch.sigmoid(b.float())
initial_state = (
torch.randn(
num_seqs,
num_v_heads,
head_v_dim,
head_k_dim,
device="cuda",
dtype=state_dtype,
generator=rng,
)
* 0.05
)
# check metadata kernel
chunk_indices, chunk_offsets = prepare_metadata_cutedsl(cu_seqlens, total_tokens)
torch.accelerator.synchronize()
expected_indices = prepare_chunk_indices(cu_seqlens, 64)
expected_offsets = prepare_chunk_offsets(cu_seqlens, 64)
total_chunks = int(expected_offsets[-1].item())
torch.testing.assert_close(chunk_offsets, expected_offsets.to(torch.int32))
torch.testing.assert_close(
chunk_indices[:total_chunks],
expected_indices,
)
ref_o, ref_state = chunk_gated_delta_rule(
q=q,
k=k,
v=v,
g=g,
beta=beta,
initial_state=initial_state,
output_final_state=True,
cu_seqlens=cu_seqlens,
use_qk_l2norm_in_kernel=False,
)
actual_core_attn_out = torch.empty(
total_tokens,
num_v_heads,
head_v_dim,
device="cuda",
dtype=dtype,
)
actual_o, actual_state = chunk_gated_delta_rule_cutedsl(
q=q,
k=k,
v=v,
g=g,
beta=beta,
initial_state=initial_state,
cu_seqlens=cu_seqlens,
chunk_indices=chunk_indices,
chunk_offsets=chunk_offsets,
core_attn_out=actual_core_attn_out,
)
torch.accelerator.synchronize()
# check main kernel
o_error = (actual_o.float() - ref_o.float()).abs()
state_error = (
actual_state.float() - ref_state.to(actual_state.dtype).float()
).abs()
assert o_error.max().item() < 2e-3
assert o_error.mean().item() < 6e-5
assert state_error.max().item() < 2e-2
assert state_error.mean().item() < 6e-4
core_attn_out_error = (
actual_core_attn_out.float() - actual_o.squeeze(0).float()
).abs()
assert core_attn_out_error.max().item() == 0
# check main kernel when core_attn_out is not passed
no_buffer_o, no_buffer_state = chunk_gated_delta_rule_cutedsl(
q=q,
k=k,
v=v,
g=g,
beta=beta,
initial_state=initial_state,
cu_seqlens=cu_seqlens,
chunk_indices=chunk_indices,
chunk_offsets=chunk_offsets,
)
torch.accelerator.synchronize()
no_buffer_o_error = (no_buffer_o.float() - ref_o.float()).abs()
no_buffer_state_error = (
no_buffer_state.float() - ref_state.to(no_buffer_state.dtype).float()
).abs()
buffer_o_error = (no_buffer_o.float() - actual_o.float()).abs()
buffer_state_error = (
no_buffer_state.float() - actual_state.to(no_buffer_state.dtype).float()
).abs()
assert no_buffer_o_error.max().item() < 2e-3
assert no_buffer_o_error.mean().item() < 6e-5
assert no_buffer_state_error.max().item() < 2e-2
assert no_buffer_state_error.mean().item() < 6e-4
assert buffer_o_error.max().item() == 0
assert buffer_state_error.max().item() == 0
def test_gdn_chunk_cutedsl_correlated_keys_stability():
"""Cover a KKT block where three BF16 Newton rounds diverge."""
total_tokens = 64
num_k_heads = 4
num_v_heads = 8
head_dim = 128
rng = torch.Generator("cuda").manual_seed(3456)
q = torch.zeros(
1,
total_tokens,
num_k_heads,
head_dim,
device="cuda",
dtype=torch.bfloat16,
)
k = torch.zeros_like(q)
q[..., 0] = 1
k[..., 0] = 1
v = torch.randn(
1,
total_tokens,
num_v_heads,
head_dim,
device="cuda",
dtype=torch.bfloat16,
generator=rng,
)
g = torch.zeros(
1,
total_tokens,
num_v_heads,
device="cuda",
dtype=torch.float32,
)
beta = torch.full_like(g, 0.9)
initial_state = torch.zeros(
1,
num_v_heads,
head_dim,
head_dim,
device="cuda",
dtype=torch.float32,
)
cu_seqlens = torch.tensor([0, total_tokens], device="cuda", dtype=torch.int32)
chunk_indices, chunk_offsets = prepare_metadata_cutedsl(cu_seqlens, total_tokens)
ref_o, ref_state = chunk_gated_delta_rule(
q=q,
k=k,
v=v,
g=g,
beta=beta,
initial_state=initial_state,
output_final_state=True,
cu_seqlens=cu_seqlens,
use_qk_l2norm_in_kernel=False,
)
actual_o, actual_state = chunk_gated_delta_rule_cutedsl(
q=q,
k=k,
v=v,
g=g,
beta=beta,
initial_state=initial_state,
cu_seqlens=cu_seqlens,
chunk_indices=chunk_indices,
chunk_offsets=chunk_offsets,
)
torch.accelerator.synchronize()
assert torch.isfinite(actual_o).all()
assert torch.isfinite(actual_state).all()
output_error = (actual_o.float() - ref_o.float()).abs()
state_error = (
actual_state.float() - ref_state.to(actual_state.dtype).float()
).abs()
assert output_error.max().item() < 1e-1
assert output_error.mean().item() < 2e-3
assert state_error.max().item() < 2e-1
assert state_error.mean().item() < 6e-4
File diff suppressed because it is too large Load Diff
@@ -1,422 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from tests.kernels.moe.modular_kernel_tools.parallel_utils import (
ProcessGroupInfo,
parallel_launch_with_config,
)
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
from vllm.forward_context import get_forward_context, set_forward_context
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.fused_moe.expert_substitution import (
ConstantExpertSubstitution,
SubstitutedRoutedExperts,
get_expert_substitution_spec,
make_expert_substitution,
)
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import (
FlashInferExperts,
)
from vllm.model_executor.models.mixtral import MixtralMoE
from vllm.platforms import current_platform
from vllm.v1.worker.workspace import init_workspace_manager
MOE_PREFIX = "model.layers.1.mlp.experts"
def _model_config(approximate_experts, **kwargs) -> SimpleNamespace:
return SimpleNamespace(
hf_config=SimpleNamespace(approximate_experts=approximate_experts),
**kwargs,
)
def _expert_substitution(targets: dict[str, dict[int, str]]) -> dict:
"""The ``compression_config.transform_config.expert_substitution`` format."""
return {
"version": 1,
"router_semantics": {
"preserve_logical_expert_ids": True,
"preserve_router_weights": True,
"renormalize_after_substitution": False,
},
"targets": {
path: {
"num_logical_experts": 5,
"weight_layout": "compact_retained_experts",
"replacements": {
str(expert_id): {
"format": "constant-v1",
"tensors": {"value": name},
}
for expert_id, name in replacements.items()
},
}
for path, replacements in targets.items()
},
}
def _compressed_model_config(expert_substitution: dict, **kwargs) -> SimpleNamespace:
return SimpleNamespace(
hf_config=SimpleNamespace(
compression_config={
"transform_config": {"expert_substitution": expert_substitution}
}
),
**kwargs,
)
def _substitution(hidden_size: int, dtype=torch.float32):
# Experts 1 and 3 of 5 are substituted; 0, 2, 4 map to physical 0, 1, 2.
return ConstantExpertSubstitution(1, 5, (1, 3), hidden_size, dtype)
def test_parse_approximate_experts():
assert get_expert_substitution_spec(None) is None
assert get_expert_substitution_spec(_model_config(None)) is None
spec = get_expert_substitution_spec(_model_config({"1": [3, 1], "2": [], 5: (0,)}))
assert spec.experts == {1: (1, 3), 5: (0,)}
assert spec.value_names == {}
text_config = SimpleNamespace(
hf_text_config=SimpleNamespace(approximate_experts={"0": [2]}),
hf_config=SimpleNamespace(),
)
assert get_expert_substitution_spec(text_config).experts == {0: (2,)}
@pytest.mark.parametrize(
"approximate_experts",
[[1, 2], {"layer": [1]}, {"1": [1, 1]}, {"1": [-1]}, {"1": ["x"]}],
)
def test_parse_approximate_experts_rejects_invalid_entries(approximate_experts):
with pytest.raises(ValueError, match="approximate_experts"):
get_expert_substitution_spec(_model_config(approximate_experts))
def test_parse_expert_substitution_with_explicit_value_names():
spec = get_expert_substitution_spec(
_compressed_model_config(
_expert_substitution(
{
MOE_PREFIX: {3: "shared", 1: "layer_1.expert_1"},
"model.layers.2.mlp.experts": {0: "shared"},
}
)
)
)
assert spec.experts == {1: (1, 3), 2: (0,)}
assert spec.value_names == {
"shared": ((1, 3), (2, 0)),
"layer_1.expert_1": ((1, 1),),
}
@pytest.mark.parametrize(
"mutate",
[
lambda raw: raw.update(version=2),
lambda raw: raw["router_semantics"].update(renormalize_after_substitution=True),
lambda raw: raw["targets"].update({"mlp.experts": raw["targets"][MOE_PREFIX]}),
lambda raw: raw["targets"][MOE_PREFIX].update(weight_layout="dense"),
lambda raw: raw["targets"][MOE_PREFIX]["replacements"]["1"].update(
format="linear-v1"
),
lambda raw: raw["targets"][MOE_PREFIX]["replacements"]["1"].pop("tensors"),
],
)
def test_parse_expert_substitution_rejects_unsupported_metadata(mutate):
raw = _expert_substitution({MOE_PREFIX: {1: "value"}})
mutate(raw)
with pytest.raises(ValueError, match="invalid expert_substitution metadata"):
get_expert_substitution_spec(_compressed_model_config(raw))
def test_parse_rejects_both_formats():
config = _compressed_model_config(_expert_substitution({MOE_PREFIX: {1: "v"}}))
config.hf_config.approximate_experts = {"1": [1]}
with pytest.raises(ValueError, match="both"):
get_expert_substitution_spec(config)
def test_make_expert_substitution_matches_layer_index():
config = _model_config({"1": [3, 1]})
substitution = make_expert_substitution(config, MOE_PREFIX, 5, 16, torch.bfloat16)
assert substitution is not None
assert substitution.layer_idx == 1
assert substitution.substituted_expert_ids == (1, 3)
assert substitution.num_compute_experts == 3
assert substitution.values.dtype == torch.bfloat16
assert [substitution.physical_expert_id(i) for i in range(5)] == [0, -1, 1, -1, 2]
assert (
make_expert_substitution(config, "model.layers.2.mlp.experts", 5, 16, None)
is None
)
assert make_expert_substitution(None, MOE_PREFIX, 5, 16, None) is None
explicit = _compressed_model_config(_expert_substitution({MOE_PREFIX: {1: "v"}}))
substitution = make_expert_substitution(explicit, MOE_PREFIX, 5, 16, None)
assert substitution is not None
assert substitution.substituted_expert_ids == (1,)
with pytest.raises(ValueError, match="exactly one layer index"):
make_expert_substitution(config, "mlp.experts", 5, 16, None)
with pytest.raises(ValueError, match="within"):
make_expert_substitution(config, MOE_PREFIX, 3, 16, None)
with pytest.raises(ValueError, match="retain at least one expert"):
make_expert_substitution(_model_config({"1": [0, 1]}), MOE_PREFIX, 2, 16, None)
def test_constant_substitution_loads_values():
substitution = _substitution(2)
substitution.weight_loader(substitution.values, torch.tensor([1.0, 2.0]), 3)
with pytest.raises(ValueError, match="missing approx_value .* \\[1\\]"):
substitution.validate_loaded_values("layer")
substitution.weight_loader(substitution.values, torch.tensor([3.0, 4.0]), 1)
substitution.validate_loaded_values("layer")
torch.testing.assert_close(
substitution.values, torch.tensor([[3.0, 4.0], [1.0, 2.0]])
)
with pytest.raises(ValueError, match="not listed"):
substitution.weight_loader(substitution.values, torch.ones(2), 2)
with pytest.raises(ValueError, match="has shape"):
substitution.weight_loader(substitution.values, torch.ones(3), 1)
@pytest.mark.parametrize("num_tokens", [0, 1, 3, 7])
@pytest.mark.parametrize("hidden_size", [2, 4])
def test_constant_substitution_transforms_routes_and_computes_side_output(
num_tokens, hidden_size
):
substitution = _substitution(2)
substitution.values.data.copy_(torch.tensor([[10.0, 20.0], [30.0, 40.0]]))
rows = torch.arange(num_tokens) % 3
topk_ids = torch.tensor([[1, 2], [4, 3], [99, -1]])[rows]
topk_weights = torch.tensor([[0.25, 0.75], [0.6, 0.4], [0.5, 0.5]])[rows]
constant_output = substitution.transform_routes(
torch.zeros(num_tokens, hidden_size), topk_weights, topk_ids
)
torch.testing.assert_close(
topk_weights, torch.tensor([[0.0, 0.75], [0.6, 0.0], [0.0, 0.0]])[rows]
)
torch.testing.assert_close(topk_ids, torch.tensor([[0, 1], [2, 0], [0, 0]])[rows])
expected = torch.zeros(num_tokens, hidden_size)
expected[:, :2] = torch.tensor([[2.5, 5.0], [12.0, 16.0], [0.0, 0.0]])[rows]
torch.testing.assert_close(constant_output, expected)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize(
"device",
[
"cpu",
pytest.param(
"cuda",
marks=pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="requires a CUDA-like platform",
),
),
],
)
def test_constant_substitution_preserves_router_weight_precision(dtype, device):
substitution = _substitution(1, dtype).to(device)
substitution.values.data.copy_(
torch.tensor([[60000.0], [-60000.0]], dtype=dtype, device=device)
)
topk_weights = torch.tensor([[0.5001, 0.4999]], dtype=torch.float32, device=device)
topk_ids = torch.tensor([[1, 3]], device=device)
# Nearly cancelling constants must retain the difference between FP32 weights.
expected = (topk_weights.double() @ substitution.values.double()).to(dtype)
actual = substitution.transform_routes(
torch.zeros(1, 1, dtype=dtype, device=device), topk_weights, topk_ids
)
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
def _bare_substituted_routed_experts(**attrs) -> SubstitutedRoutedExperts:
layer = SubstitutedRoutedExperts.__new__(SubstitutedRoutedExperts)
torch.nn.Module.__init__(layer)
layer.expert_substitution = _substitution(2)
for name, value in attrs.items():
setattr(layer, name, value)
return layer
@pytest.mark.parametrize("tp_rank", [0, 1])
@pytest.mark.parametrize("padded_input", [False, True])
def test_substituted_routed_experts_adds_constant_on_one_tp_rank(tp_rank, padded_input):
def apply(*, layer, x, topk_weights, topk_ids, **kwargs):
torch.testing.assert_close(topk_weights, torch.tensor([[0.0, 0.75]]))
torch.testing.assert_close(topk_ids, torch.tensor([[0, 1]]))
return torch.ones(x.shape[0], 2)
layer = _bare_substituted_routed_experts(
moe_config=SimpleNamespace(tp_rank=tp_rank),
quant_method=SimpleNamespace(is_monolithic=False, apply=apply),
)
layer.expert_substitution.values.data.copy_(
torch.tensor([[10.0, 20.0], [30.0, 40.0]])
)
actual = layer.forward_modular(
torch.zeros(1, 4 if padded_input else 2),
torch.tensor([[0.25, 0.75]]),
torch.tensor([[1, 2]]),
)
expected = torch.ones(1, 2)
if tp_rank == 0:
expected += torch.tensor([[2.5, 5.0]])
torch.testing.assert_close(actual, expected)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like platform"
)
def test_standard_fused_moe_model_discovers_substitution(dist_init):
vllm_config = VllmConfig()
vllm_config.model_config = _model_config(
{"0": [1]}, dtype=torch.bfloat16, is_moe=True
)
vllm_config.kernel_config.moe_backend = "triton"
with set_current_vllm_config(vllm_config):
moe = MixtralMoE(
num_experts=4,
top_k=2,
hidden_size=256,
intermediate_size=512,
params_dtype=torch.bfloat16,
prefix="model.layers.0.block_sparse_moe",
)
routed_experts = moe.experts.routed_experts
assert isinstance(routed_experts, SubstitutedRoutedExperts)
assert routed_experts.expert_substitution.substituted_expert_ids == (1,)
assert routed_experts.w13_weight.shape[0] == 3
assert moe.experts.moe_config.require_decomposed_backend
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like platform"
)
@pytest.mark.parametrize(
"moe_backend",
[
pytest.param("triton", id="triton"),
pytest.param(
"flashinfer_cutlass",
id="flashinfer-cutlass",
marks=pytest.mark.skipif(
not FlashInferExperts._supports_current_device(),
reason="FlashInfer CUTLASS is unavailable on this platform",
),
),
],
)
def test_fused_moe_combines_compact_experts_and_substitution(dist_init, moe_backend):
hidden_size = 256
vllm_config = VllmConfig()
vllm_config.model_config = _model_config(
{"0": [1, 3]}, dtype=torch.bfloat16, is_moe=True
)
vllm_config.compilation_config.static_forward_context = {}
vllm_config.kernel_config.moe_backend = moe_backend
with set_current_vllm_config(vllm_config), set_forward_context(None, vllm_config):
init_workspace_manager(torch.accelerator.current_device_index())
layer = FusedMoEFactory(
num_experts=4,
top_k=2,
hidden_size=hidden_size,
intermediate_size=512,
params_dtype=torch.bfloat16,
prefix="model.layers.0.mlp.experts",
renormalize=False,
).cuda()
substitution = layer.routed_experts.expert_substitution
assert layer.routed_experts.w13_weight.shape[0] == 2
with torch.no_grad():
layer.routed_experts.w13_weight.normal_(0, 0.01)
layer.routed_experts.w2_weight.normal_(0, 0.01)
substitution.values.normal_(0, 0.01)
layer._quant_method.process_weights_after_loading(layer.routed_experts)
hidden_states = torch.randn(8, hidden_size, dtype=torch.bfloat16, device="cuda")
router_logits = torch.randn(8, 4, dtype=torch.float32, device="cuda")
topk_weights, topk_ids = layer.router.select_experts(
hidden_states.clone(), router_logits
)
expected = substitution.transform_routes(hidden_states, topk_weights, topk_ids)
expected += layer._quant_method.apply(
layer=layer.routed_experts,
x=hidden_states.clone(),
topk_weights=topk_weights,
topk_ids=topk_ids,
shared_experts=None,
shared_experts_input=None,
)
get_forward_context().all_moe_layers = None
actual = layer(hidden_states.clone(), router_logits)
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
def _tp_substitution_worker(
process_group: ProcessGroupInfo,
vllm_config: VllmConfig,
_cpu_group,
) -> None:
with set_forward_context(None, vllm_config):
init_workspace_manager(process_group.local_rank)
layer = FusedMoEFactory(
num_experts=4,
top_k=1,
hidden_size=256,
intermediate_size=512,
params_dtype=torch.bfloat16,
prefix="model.layers.0.mlp.experts",
renormalize=True,
)
with torch.no_grad():
layer.routed_experts.w13_weight.zero_()
layer.routed_experts.w2_weight.zero_()
layer.routed_experts.expert_substitution.values.fill_(0.25)
layer._quant_method.process_weights_after_loading(layer.routed_experts)
hidden_states = torch.zeros(2, 256, dtype=torch.bfloat16)
router_logits = torch.tensor(
[[-100.0, 100.0, -100.0, -100.0]] * 2,
dtype=torch.float32,
)
get_forward_context().all_moe_layers = None
actual = layer(hidden_states, router_logits)
torch.testing.assert_close(
actual, torch.full_like(actual, 0.25), atol=0, rtol=0
)
@pytest.mark.skipif(
not current_platform.is_cuda_alike() or current_platform.device_count() < 2,
reason="requires two CUDA-like devices",
)
def test_constant_substitution_is_added_once_with_tp2():
vllm_config = VllmConfig(parallel_config=ParallelConfig(tensor_parallel_size=2))
vllm_config.model_config = _model_config(
{"0": [1]},
dtype=torch.bfloat16,
is_moe=True,
sleep_mode_offload_cudagraph=False,
)
vllm_config.compilation_config.static_forward_context = {}
vllm_config.kernel_config.moe_backend = "triton"
parallel_launch_with_config(2, _tp_substitution_worker, vllm_config, None)
File diff suppressed because it is too large Load Diff
-57
View File
@@ -1,57 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.model_executor.models.gemma4 import (
gemma4_fused_routing_kernel_triton,
gemma4_routing_function_torch,
)
def sort_by_id(w, ids):
order = ids.argsort(dim=-1)
return w.gather(1, order), ids.gather(1, order)
# Gemma4 MoE Model has context length of 250K
# the minus 1 is to ensure that edge cases are tested
@pytest.mark.parametrize("num_tokens", [1, 2, 2048, 250000])
@pytest.mark.parametrize("num_experts", [128]) # gemma4 moe experts
@pytest.mark.parametrize("topk", [8]) # gemma4 topk
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half, torch.float32])
def test_gemma4_routing_kernel_triton(
num_tokens: int,
num_experts: int,
topk: int,
dtype: torch.dtype,
):
torch.manual_seed(0)
gating = torch.randn(num_tokens, num_experts, dtype=dtype, device="cuda")
scales = torch.rand(num_experts, dtype=torch.float32, device="cuda")
ref_w, ref_ids = gemma4_routing_function_torch(gating, topk, scales)
tri_w, tri_ids = gemma4_fused_routing_kernel_triton(gating, topk, scales)
# Sort by expert id — to remove tie-breaking differences
ref_ws, ref_is = sort_by_id(ref_w, ref_ids)
tri_ws, tri_is = sort_by_id(tri_w, tri_ids)
ids_match = (ref_is == tri_is).all().item()
weights_match = torch.allclose(ref_ws, tri_ws, atol=1e-2, rtol=1e-2)
all_match = ids_match and weights_match
max_err = (ref_ws - tri_ws).abs().max().item()
print(
f"T={num_tokens:5d} E={num_experts:4d} K={topk} "
f"{str(dtype).split('.')[-1]:7s} ids={ids_match} max_Δweight={max_err:.2e}"
)
if not all_match:
bad = (ref_is != tri_is).any(dim=-1).nonzero(as_tuple=True)[0]
if len(bad):
r = bad[0].item()
print(
f" first bad row {r}: ref_ids={ref_ids[r].tolist()} "
f"tri_ids={tri_ids[r].tolist()}"
)
assert all_match
File diff suppressed because it is too large Load Diff
-541
View File
@@ -1,541 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm-specific tests for FP8 quantization and kernels."""
import importlib
import math
import warnings
import pytest
import torch
import vllm._custom_ops as ops
from tests.kernels.quant_utils import (
ref_dynamic_per_tensor_fp8_quant,
ref_dynamic_per_token_quant,
)
from tests.kernels.utils import _assert_deterministic
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(), reason="ROCm-specific tests"
)
DEVICE = "cuda"
FP8_DTYPE = current_platform.fp8_dtype()
fp8_only = pytest.mark.skipif(
not current_platform.supports_fp8(), reason="ROCm FP8 only"
)
ENV_DEFAULTS = {
"VLLM_ROCM_FP8_PADDING": True,
"VLLM_ROCM_USE_AITER_FP8BMM": False,
"VLLM_ROCM_FP8_MFMA_PAGE_ATTN": False,
}
SCALED_MM_CASES = [
pytest.param((1, 128, 128), 0.05, 0.15, 0.95, id="1x128x128"),
pytest.param((4, 256, 512), 0.10, 0.30, 0.96, id="4x256x512"),
pytest.param((8, 512, 1024), 0.20, 0.50, 0.99, id="8x512x1024"),
]
def _reload_envs():
import vllm.envs as envs
return importlib.reload(envs)
@pytest.fixture(autouse=True)
def _restore_rocm_env_state():
"""Restore global env + AITER flag state after every test.
Tests here reload ``vllm.envs`` (a module singleton) and refresh the
cached ``rocm_aiter_ops`` env flags. Restoring in a fixture teardown
guarantees cleanup even when an assertion fails mid-test, preventing
state from leaking into other tests in the same worker.
"""
yield
_reload_envs()
from vllm._aiter_ops import rocm_aiter_ops
rocm_aiter_ops.refresh_env_variables()
def _quantile(flat: torch.Tensor, q: float) -> float:
if flat.numel() == 1:
return flat.item()
return torch.quantile(flat, q).item()
def _format_observed_rate(count: int, total: int) -> str:
return f"{count / total:.4%} ({count}/{total})"
def _format_allowed_rate(rate: float, total: int) -> str:
allowed_count = math.floor(rate * total)
return f"{rate:.4%} (<= {allowed_count}/{total})"
def _assert_abs_error_budget(
actual: torch.Tensor,
expected: torch.Tensor,
*,
label: str,
tight_atol: float,
max_atol: float,
pass_rate: float,
) -> None:
diff = (actual.float() - expected.float()).abs().flatten()
total = diff.numel()
within_tight_count = int((diff <= tight_atol).sum().item())
fail_count = total - within_tight_count
above_max_count = int((diff > max_atol).sum().item())
within_tight = within_tight_count / total
allowed_fail_rate = 1.0 - pass_rate
above_max = above_max_count / total
max_fail_rate = 0.0
max_diff = diff.max().item()
mean_diff = diff.mean().item()
p99 = _quantile(diff, 0.99)
p999 = _quantile(diff, 0.999)
msg = (
f"[rocm_fp8] {label}: abs<={tight_atol} pass={within_tight:.4%} "
f"({within_tight_count}/{total}) "
f"fail={_format_observed_rate(fail_count, total)} "
f"allowed_fail={_format_allowed_rate(allowed_fail_rate, total)} "
f"abs>{max_atol}={_format_observed_rate(above_max_count, total)} "
f"allowed_above_max={_format_allowed_rate(max_fail_rate, total)} "
f"max_diff={max_diff:.6g} mean_diff={mean_diff:.6g} "
f"p99={p99:.6g} p999={p999:.6g}"
)
print(msg)
if within_tight < 1.0:
warnings.warn(msg, stacklevel=2)
assert within_tight >= pass_rate, msg
assert above_max <= max_fail_rate, msg
def _assert_rel_error_budget(
actual: torch.Tensor,
expected: torch.Tensor,
*,
label: str,
tight_rtol: float,
max_rtol: float,
pass_rate: float,
max_fail_rate: float,
) -> None:
rel = (
(actual.float() - expected.float()).abs()
/ expected.float().abs().clamp_min(1e-5)
).flatten()
total = rel.numel()
within_tight_count = int((rel <= tight_rtol).sum().item())
fail_count = total - within_tight_count
above_max_count = int((rel > max_rtol).sum().item())
within_tight = within_tight_count / total
allowed_fail_rate = 1.0 - pass_rate
above_max = above_max_count / total
max_rel = rel.max().item()
mean_rel = rel.mean().item()
p99 = _quantile(rel, 0.99)
p999 = _quantile(rel, 0.999)
msg = (
f"[rocm_fp8] {label}: rel<={tight_rtol} pass={within_tight:.4%} "
f"({within_tight_count}/{total}) "
f"fail={_format_observed_rate(fail_count, total)} "
f"allowed_fail={_format_allowed_rate(allowed_fail_rate, total)} "
f"rel>{max_rtol}={_format_observed_rate(above_max_count, total)} "
f"allowed_above_max={_format_allowed_rate(max_fail_rate, total)} "
f"max_rel={max_rel:.6g} mean_rel={mean_rel:.6g} "
f"p99={p99:.6g} p999={p999:.6g}"
)
print(msg)
if within_tight < 1.0:
warnings.warn(msg, stacklevel=2)
assert within_tight >= pass_rate, msg
assert above_max <= max_fail_rate, msg
def _ref_paged_attention(
query: torch.Tensor,
key_cache: torch.Tensor,
value_cache: torch.Tensor,
*,
query_lens: list[int],
kv_lens: list[int],
block_tables: torch.Tensor,
scale: float,
) -> torch.Tensor:
block_tables_np = block_tables.cpu().numpy()
_, block_size, num_kv_heads, head_size = key_cache.shape
outputs = []
start_idx = 0
for i, query_len in enumerate(query_lens):
kv_len = kv_lens[i]
q = query[start_idx : start_idx + query_len] * scale
num_kv_blocks = (kv_len + block_size - 1) // block_size
block_indices = block_tables_np[i, :num_kv_blocks]
k = key_cache[block_indices].view(-1, num_kv_heads, head_size)[:kv_len]
v = value_cache[block_indices].view(-1, num_kv_heads, head_size)[:kv_len]
if q.shape[1] != k.shape[1]:
repeat = q.shape[1] // k.shape[1]
k = torch.repeat_interleave(k, repeat, dim=1)
v = torch.repeat_interleave(v, repeat, dim=1)
attn = torch.einsum("qhd,khd->hqk", q, k).float()
mask = torch.triu(
torch.ones(query_len, kv_len, device=q.device),
diagonal=kv_len - query_len + 1,
).bool()
attn.masked_fill_(mask, float("-inf"))
attn = torch.softmax(attn, dim=-1).to(v.dtype)
outputs.append(torch.einsum("hqk,khd->qhd", attn, v))
start_idx += query_len
return torch.cat(outputs, dim=0)
def test_rocm_fp8_env_defaults(monkeypatch):
with monkeypatch.context() as mp:
for env_name in ENV_DEFAULTS:
mp.delenv(env_name, raising=False)
envs = _reload_envs()
for env_name, expected in ENV_DEFAULTS.items():
assert getattr(envs, env_name) is expected
@pytest.mark.parametrize(
("env_name", "raw_value", "expected"),
[
("VLLM_ROCM_FP8_PADDING", "0", False),
("VLLM_ROCM_FP8_PADDING", "1", True),
("VLLM_ROCM_USE_AITER_FP8BMM", "0", False),
("VLLM_ROCM_USE_AITER_FP8BMM", "1", True),
("VLLM_ROCM_FP8_MFMA_PAGE_ATTN", "0", False),
("VLLM_ROCM_FP8_MFMA_PAGE_ATTN", "1", True),
],
)
def test_rocm_fp8_env_overrides(env_name, raw_value, expected, monkeypatch):
with monkeypatch.context() as mp:
mp.setenv(env_name, raw_value)
envs = _reload_envs()
assert getattr(envs, env_name) is expected
@fp8_only
def test_maybe_pad_fp8_weight_respects_env(monkeypatch):
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
_maybe_pad_fp8_weight,
)
weight = torch.randn(64, 512, device=DEVICE).to(FP8_DTYPE)
with monkeypatch.context() as mp:
mp.setenv("VLLM_ROCM_FP8_PADDING", "1")
_reload_envs()
padded = _maybe_pad_fp8_weight(weight)
assert padded.shape == weight.shape
assert padded.dtype == weight.dtype
assert padded.data_ptr() != weight.data_ptr()
torch.testing.assert_close(padded.float(), weight.float())
with monkeypatch.context() as mp:
mp.setenv("VLLM_ROCM_FP8_PADDING", "0")
_reload_envs()
unpadded = _maybe_pad_fp8_weight(weight)
assert unpadded.data_ptr() == weight.data_ptr()
@fp8_only
@pytest.mark.parametrize(
("use_aiter", "fp8bmm_enabled", "expected"),
[
(True, True, True),
(True, False, False),
(False, True, False),
],
)
def test_aiter_fp8bmm_enabled_api_respects_env(
use_aiter, fp8bmm_enabled, expected, monkeypatch
):
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
assert is_aiter_found_and_supported()
with monkeypatch.context() as mp:
mp.setenv("VLLM_ROCM_USE_AITER", "1" if use_aiter else "0")
mp.setenv("VLLM_ROCM_USE_AITER_FP8BMM", "1" if fp8bmm_enabled else "0")
_reload_envs()
rocm_aiter_ops.refresh_env_variables()
assert rocm_aiter_ops.is_fp8bmm_enabled() is expected
@fp8_only
@pytest.mark.parametrize(
("shape", "tight_atol", "max_atol", "pass_rate"), SCALED_MM_CASES
)
def test_fp8_scaled_mm_matches_dequantized_reference(
shape, tight_atol, max_atol, pass_rate
):
torch.manual_seed(0)
batch, m_dim, k_dim = shape
n_dim = k_dim
a = torch.randn(batch * m_dim, k_dim, device=DEVICE)
b = torch.randn(n_dim, k_dim, device=DEVICE)
a_fp8, scale_a = ref_dynamic_per_tensor_fp8_quant(a)
b_fp8, scale_b = ref_dynamic_per_tensor_fp8_quant(b)
out = torch._scaled_mm(
a_fp8,
b_fp8.t(),
out_dtype=torch.bfloat16,
scale_a=scale_a,
scale_b=scale_b,
)
a_dq = a_fp8.float() * scale_a.float()
b_dq = b_fp8.float() * scale_b.float()
ref = torch.mm(a_dq, b_dq.t())
assert out.shape == (batch * m_dim, n_dim)
assert out.dtype == torch.bfloat16
assert not torch.isnan(out).any()
_assert_abs_error_budget(
out.float(),
ref,
label=f"scaled_mm shape={shape}",
tight_atol=tight_atol,
max_atol=max_atol,
pass_rate=pass_rate,
)
@fp8_only
def test_fp8_scaled_mm_determinism():
torch.manual_seed(5)
m_dim, k_dim, n_dim = 64, 128, 128
a = torch.randn(m_dim, k_dim, device=DEVICE)
b = torch.randn(n_dim, k_dim, device=DEVICE)
a_fp8, scale_a = ref_dynamic_per_tensor_fp8_quant(a)
b_fp8, scale_b = ref_dynamic_per_tensor_fp8_quant(b)
def run_gemm():
return torch._scaled_mm(
a_fp8,
b_fp8.t(),
out_dtype=torch.bfloat16,
scale_a=scale_a,
scale_b=scale_b,
)
_assert_deterministic(run_gemm, n_runs=4)
@fp8_only
def test_aiter_fp8_kv_attention_matches_bf16_reference():
import aiter
from vllm._aiter_ops import is_aiter_found_and_supported
from vllm.v1.attention.backends.rocm_aiter_fa import cp_mha_gather_cache
assert is_aiter_found_and_supported()
set_random_seed(10)
num_q_heads, num_kv_heads = 8, 8
head_size = 128
query_len, kv_len = 4, 128
block_size = 16
num_blocks = 2048
scale = head_size**-0.5
query = torch.randn(
query_len, num_q_heads, head_size, dtype=torch.bfloat16, device=DEVICE
)
key_cache_fp8 = torch.clamp(
torch.randn(num_blocks, block_size, num_kv_heads, head_size, device=DEVICE),
min=-1.0,
max=1.0,
).to(FP8_DTYPE)
value_cache_fp8 = torch.clamp(
torch.randn(num_blocks, block_size, num_kv_heads, head_size, device=DEVICE),
min=-1.0,
max=1.0,
).to(FP8_DTYPE)
cu_query_lens = torch.tensor([0, query_len], dtype=torch.int32, device=DEVICE)
cu_seq_lens = torch.tensor([0, kv_len], dtype=torch.int32, device=DEVICE)
block_tables = torch.randint(
0,
num_blocks,
(1, (kv_len + block_size - 1) // block_size),
dtype=torch.int32,
device=DEVICE,
)
token_to_batch = torch.zeros(kv_len, dtype=torch.int32, device=DEVICE)
seq_starts = torch.zeros(1, dtype=torch.int32, device=DEVICE)
gathered_key = torch.empty(
kv_len, num_kv_heads, head_size, dtype=torch.bfloat16, device=DEVICE
)
gathered_value = torch.empty_like(gathered_key)
cp_mha_gather_cache(
key_cache=key_cache_fp8,
value_cache=value_cache_fp8,
key=gathered_key,
value=gathered_value,
block_tables=block_tables,
k_scales=torch.ones(1, dtype=torch.float32, device=DEVICE),
v_scales=torch.ones(1, dtype=torch.float32, device=DEVICE),
cu_seqlens_kv=cu_seq_lens,
token_to_batch=token_to_batch,
seq_starts=seq_starts,
dequant=True,
kv_cache_layout="NHD",
total_tokens=kv_len,
)
output = torch.empty_like(query)
aiter.flash_attn_varlen_func(
q=query,
k=gathered_key,
v=gathered_value,
cu_seqlens_q=cu_query_lens,
cu_seqlens_k=cu_seq_lens,
max_seqlen_q=query_len,
max_seqlen_k=kv_len,
min_seqlen_q=1,
dropout_p=0.0,
softmax_scale=scale,
causal=True,
window_size=(-1, -1),
alibi_slopes=None,
return_lse=False,
out=output,
)
ref = _ref_paged_attention(
query=query,
key_cache=key_cache_fp8.to(torch.bfloat16),
value_cache=value_cache_fp8.to(torch.bfloat16),
query_lens=[query_len],
kv_lens=[kv_len],
block_tables=block_tables,
scale=scale,
)
_assert_abs_error_budget(
output.float(),
ref.float(),
label="aiter_fp8_kv_attention",
tight_atol=0.15,
max_atol=0.25,
pass_rate=0.98,
)
@fp8_only
@pytest.mark.parametrize("shape", [(128, 256), (512, 1024), (1024, 4096)])
def test_fp8_per_tensor_quant_matches_reference(shape):
torch.manual_seed(0)
x = torch.randn(*shape, dtype=torch.bfloat16, device=DEVICE)
out, scale = ops.scaled_fp8_quant(x)
ref_out, ref_scale = ref_dynamic_per_tensor_fp8_quant(x)
assert out.shape == x.shape
assert out.dtype == FP8_DTYPE
torch.testing.assert_close(scale, ref_scale, rtol=1e-3, atol=1e-4)
code_match_rate = (
(out.view(torch.uint8) == ref_out.view(torch.uint8)).float().mean().item()
)
assert code_match_rate >= 0.999, (
f"FP8 codes match reference only {code_match_rate:.4%} (< 99.9%)"
)
dequant = out.float() * scale.float()
_assert_rel_error_budget(
dequant,
x.float(),
label=f"per_tensor_quant shape={shape}",
tight_rtol=0.06,
max_rtol=0.60,
pass_rate=0.99,
max_fail_rate=1e-4,
)
@fp8_only
def test_aiter_group_fp8_quant_roundtrip():
from aiter import dtypes
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
assert is_aiter_found_and_supported()
torch.manual_seed(0)
m_dim, n_dim = 64, 4096
group_size = 128
x = torch.randn(m_dim, n_dim, dtype=torch.bfloat16, device=DEVICE)
out, scales = rocm_aiter_ops.group_fp8_quant(x, group_size)
assert out.dtype == dtypes.fp8
assert scales.dtype == torch.float32
scales_expanded = scales.repeat_interleave(group_size, dim=1)[:, :n_dim]
dequant = out.float() * scales_expanded
_assert_rel_error_budget(
dequant,
x.float(),
label="aiter_group_fp8_quant",
tight_rtol=0.06,
max_rtol=0.30,
pass_rate=0.99,
max_fail_rate=1e-4,
)
@fp8_only
def test_fp8_per_token_quant_matches_reference():
torch.manual_seed(0)
x = torch.randn(32, 4096, dtype=torch.bfloat16, device=DEVICE)
out, scale = ops.scaled_fp8_quant(x, scale=None, use_per_token_if_dynamic=True)
ref_out, ref_scale = ref_dynamic_per_token_quant(x, FP8_DTYPE)
assert out.shape == x.shape
assert out.dtype == FP8_DTYPE
assert scale.shape == (x.shape[0], 1)
torch.testing.assert_close(scale, ref_scale, rtol=1e-3, atol=1e-4)
code_match_rate = (
(out.view(torch.uint8) == ref_out.view(torch.uint8)).float().mean().item()
)
assert code_match_rate >= 0.999, (
f"FP8 codes match reference only {code_match_rate:.4%} (< 99.9%)"
)
dequant = out.float() * scale.float()
_assert_rel_error_budget(
dequant,
x.float(),
label="per_token_quant",
tight_rtol=0.06,
max_rtol=0.30,
pass_rate=0.99,
max_fail_rate=1e-4,
)
@@ -1,60 +0,0 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Check that DeepSeek V4 and the JIT monitor leave the HIP symbols alone.
TileLang ships a libhip_stub.so that takes over `hipFree` in the global symbol
table once it is loaded, so neither the model import nor the JIT monitor may
pull TileLang in on ROCm. Both claims are about process-global state, which is
why this runs in an interpreter of its own.
"""
import ctypes
import sys
from vllm.model_executor.layers import mhc # noqa: F401
from vllm.models import deepseek_v4 # noqa: F401
from vllm.utils import jit_monitor
class DlInfo(ctypes.Structure):
_fields_ = [
("dli_fname", ctypes.c_char_p),
("dli_fbase", ctypes.c_void_p),
("dli_sname", ctypes.c_char_p),
("dli_saddr", ctypes.c_void_p),
]
def _must_not_run() -> None:
raise AssertionError("TileLang JIT monitor must not run on ROCm")
jit_monitor._active = False
jit_monitor._setup_triton_autotuning_print = lambda: None
jit_monitor._setup_triton_jit_hook = lambda: None
jit_monitor._setup_cutedsl_jit_hook = lambda: None
jit_monitor._setup_tilelang_jit_hook = _must_not_run
jit_monitor.activate()
imported = [
name for name in sys.modules if name == "tilelang" or name.startswith("tilelang.")
]
assert not imported, f"TileLang was imported: {imported}"
libdl = ctypes.CDLL("libdl.so.2")
dlsym = libdl.dlsym
dlsym.argtypes = [ctypes.c_void_p, ctypes.c_char_p]
dlsym.restype = ctypes.c_void_p
dladdr = libdl.dladdr
dladdr.argtypes = [ctypes.c_void_p, ctypes.POINTER(DlInfo)]
dladdr.restype = ctypes.c_int
address = dlsym(None, b"hipFree")
assert address is not None, "hipFree is not available in the global symbol table"
info = DlInfo()
assert dladdr(address, ctypes.byref(info))
source = info.dli_fname.decode() if info.dli_fname else "<unknown>"
assert "libhip_stub.so" not in source, source
assert "libamdhip64.so" in source, source
print("OK")
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
-185
View File
@@ -1,185 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm import _custom_ops as ops
NUM_TOKENS = [1, 4, 16, 64, 128]
NUM_HEADS = [128]
NOPE_DIM = [512]
ROPE_DIM = [64]
DTYPES = [torch.bfloat16, torch.float16]
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("nope_dim", NOPE_DIM)
@pytest.mark.parametrize("rope_dim", ROPE_DIM)
@pytest.mark.parametrize("dtype", DTYPES)
def test_concat_mla_q_contiguous(num_tokens, num_heads, nope_dim, rope_dim, dtype):
"""Test with contiguous inputs (standard layout)."""
torch.manual_seed(42)
ql_nope = torch.randn(num_tokens, num_heads, nope_dim, dtype=dtype, device="cuda")
q_pe = torch.randn(num_tokens, num_heads, rope_dim, dtype=dtype, device="cuda")
ref = torch.cat((ql_nope, q_pe), dim=-1)
q_out = torch.empty(
num_tokens, num_heads, nope_dim + rope_dim, dtype=dtype, device="cuda"
)
ops.concat_mla_q(ql_nope, q_pe, q_out)
torch.testing.assert_close(q_out, ref, atol=0, rtol=0)
def test_concat_mla_q_padded_output_heads():
"""The fused concat can write directly into a head-padded workspace."""
ql_nope = torch.randn(4, 8, 512, dtype=torch.bfloat16, device="cuda")
q_pe = torch.randn(4, 8, 64, dtype=torch.bfloat16, device="cuda")
q_out = torch.empty(4, 64, 576, dtype=torch.bfloat16, device="cuda")
ops.concat_mla_q(ql_nope, q_pe, q_out)
torch.testing.assert_close(q_out[:, :8], torch.cat((ql_nope, q_pe), dim=-1))
@pytest.mark.parametrize("num_tokens", [t for t in NUM_TOKENS if t > 1])
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("nope_dim", NOPE_DIM)
@pytest.mark.parametrize("rope_dim", ROPE_DIM)
@pytest.mark.parametrize("dtype", DTYPES)
def test_concat_mla_q_transposed_nope(num_tokens, num_heads, nope_dim, rope_dim, dtype):
"""Test with transposed nope input (simulates BMM output after transpose).
In the real code path, mqa_ql_nope is the result of:
torch.bmm(q_nope, W_UK_T) # [N, B, L]
.transpose(0, 1) # [B, N, L] — non-contiguous!
"""
torch.manual_seed(42)
nope_raw = torch.randn(num_heads, num_tokens, nope_dim, dtype=dtype, device="cuda")
ql_nope = nope_raw.transpose(0, 1) # [B, N, L], non-contiguous
assert not ql_nope.is_contiguous()
q_pe = torch.randn(num_tokens, num_heads, rope_dim, dtype=dtype, device="cuda")
ref = torch.cat((ql_nope, q_pe), dim=-1)
q_out = torch.empty(
num_tokens, num_heads, nope_dim + rope_dim, dtype=dtype, device="cuda"
)
ops.concat_mla_q(ql_nope, q_pe, q_out)
torch.testing.assert_close(q_out, ref, atol=0, rtol=0)
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("dtype", DTYPES)
def test_concat_mla_q_split_rope(num_tokens, num_heads, dtype):
"""Test with rope from a split (simulates the actual code path).
In the real code path, q_pe comes from:
mqa_q.split([qk_nope_head_dim, qk_rope_head_dim], dim=-1)
which creates a non-contiguous view with stride(1) != rope_dim.
"""
torch.manual_seed(42)
nope_dim = 512
rope_dim = 64
orig_dim = 128 + 64 # original q before absorption: [B, N, 192]
# Simulate split from original q tensor
q_orig = torch.randn(num_tokens, num_heads, orig_dim, dtype=dtype, device="cuda")
q_nope_orig, q_pe = q_orig.split([128, 64], dim=-1)
# q_pe is non-contiguous: stride(1) = 192, not 64
assert q_pe.stride(1) == orig_dim
assert q_pe.stride(2) == 1 # but innermost is fine
# Simulate absorbed nope (contiguous, different size)
ql_nope = torch.randn(num_tokens, num_heads, nope_dim, dtype=dtype, device="cuda")
ref = torch.cat((ql_nope, q_pe), dim=-1)
q_out = torch.empty(
num_tokens, num_heads, nope_dim + rope_dim, dtype=dtype, device="cuda"
)
ops.concat_mla_q(ql_nope, q_pe, q_out)
torch.testing.assert_close(q_out, ref, atol=0, rtol=0)
def test_concat_mla_q_zero_tokens():
"""Test with zero tokens (edge case)."""
ql_nope = torch.empty(0, 128, 512, dtype=torch.bfloat16, device="cuda")
q_pe = torch.empty(0, 128, 64, dtype=torch.bfloat16, device="cuda")
q_out = torch.empty(0, 128, 576, dtype=torch.bfloat16, device="cuda")
ops.concat_mla_q(ql_nope, q_pe, q_out)
@pytest.mark.parametrize("num_tokens", [1, 64])
def test_concat_mla_q_values_preserved(num_tokens):
"""Verify exact bit-level preservation (no computation, pure copy).
Compares raw int16 bits to avoid NaN != NaN issues from IEEE 754.
"""
nope_dim, rope_dim = 512, 64
# Use specific bit patterns (stay in int16 for bit-exact comparison)
ql_nope_bits = torch.arange(
num_tokens * 128 * nope_dim, dtype=torch.int16, device="cuda"
).view(num_tokens, 128, nope_dim)
q_pe_bits = torch.arange(
num_tokens * 128 * rope_dim, dtype=torch.int16, device="cuda"
).view(num_tokens, 128, rope_dim)
ql_nope = ql_nope_bits.view(torch.bfloat16)
q_pe = q_pe_bits.view(torch.bfloat16)
q_out = torch.empty(
num_tokens, 128, nope_dim + rope_dim, dtype=torch.bfloat16, device="cuda"
)
ops.concat_mla_q(ql_nope, q_pe, q_out)
out_bits = q_out.view(torch.int16)
assert torch.equal(out_bits[..., :nope_dim], ql_nope_bits)
assert torch.equal(out_bits[..., nope_dim:], q_pe_bits)
@pytest.mark.parametrize("rope_dim,transposed", [(0, False), (0, True), (64, False)])
@pytest.mark.parametrize("enabled", [False, True])
def test_concat_mla_q_fp8_nope_and_rope(rope_dim, transposed, enabled):
"""FP8 query packing preserves values with empty RoPE and strided BMM output."""
from vllm.config import CompilationConfig, VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.attention.mla_attention import _DecodeConcatQuantFP8
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
get_fp8_min_max,
)
from vllm.platforms import current_platform
torch.manual_seed(42)
shape = (8, 17, 512) if transposed else (17, 8, 512)
ql = torch.randn(shape, dtype=torch.bfloat16, device="cuda") * 128
if transposed:
ql = ql.transpose(0, 1)
qp = torch.randn(17, 8, rope_dim, dtype=torch.bfloat16, device="cuda") * 128
scale = torch.tensor(0.3, device="cuda", dtype=torch.float32)
joined = torch.cat((ql, qp), dim=-1)
fp8_min, fp8_max = get_fp8_min_max()
expected = (joined.float() * scale.reciprocal()).clamp(fp8_min, fp8_max)
expected = expected.to(current_platform.fp8_dtype())
custom_ops = ["none"] + (["+mla_decode_concat_quant_fp8"] if enabled else [])
config = VllmConfig(
compilation_config=CompilationConfig(mode=0, custom_ops=custom_ops)
)
with set_current_vllm_config(config):
quant = _DecodeConcatQuantFP8(static=True, group_shape=GroupShape.PER_TENSOR)
actual = quant(ql, qp, scale)
assert actual.dtype == current_platform.fp8_dtype()
torch.testing.assert_close(actual.float(), expected.float(), atol=0, rtol=0)
assert actual.is_contiguous()
File diff suppressed because it is too large Load Diff
@@ -1,79 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Layout plumbing for DeepSeek V4.1 mega attention.
The kernel's Q and O layouts are produced by permuting ``wq_b`` rows and
``wo_a`` columns in their weight loaders, so what has to hold is that the
permuted GEMMs agree with a reference that permutes the activation instead.
"""
import torch
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.models.deepseek_v41.common.ops.fused_layout import (
WV_GROUP_SIZE,
o_fused_chunk_permutation,
o_fused_permutation,
permute_on_load,
permute_q_to_fused,
q_fused_permutation,
)
HEAD_DIM = 512
def _load(checkpoint: torch.Tensor, perm: torch.Tensor, dim: int) -> torch.Tensor:
"""Load ``checkpoint`` into a fresh parameter through a permuting loader."""
param = torch.nn.Parameter(torch.empty_like(checkpoint), requires_grad=False)
param.weight_loader = default_weight_loader # type: ignore[attr-defined]
permute_on_load(param, perm, dim)
param.weight_loader(param, checkpoint) # type: ignore[attr-defined]
return param.data
def test_permuted_wq_b_gemm_matches_permuted_activation():
"""Permuting wq_b's rows makes the Q GEMM emit the fused layout directly."""
torch.manual_seed(0)
num_heads, q_lora_rank, num_tokens = 16, 64, 5
weight = torch.randn(num_heads * HEAD_DIM, q_lora_rank)
# One scale row per weight row, as an MXFP8 wq_b shard carries.
scale = torch.randint(
0, 256, (num_heads * HEAD_DIM, q_lora_rank // 32), dtype=torch.uint8
)
qr = torch.randn(num_tokens, q_lora_rank)
standard = (qr @ weight.T).view(num_tokens, num_heads, HEAD_DIM)
expected = permute_q_to_fused(standard)
perm = q_fused_permutation(num_heads, HEAD_DIM)
loaded_weight = _load(weight, perm, dim=0)
fused = (qr @ loaded_weight.T).view(num_tokens, num_heads, HEAD_DIM)
torch.testing.assert_close(fused, expected)
# The scale has to follow its row, or dequant reads another row's scale.
torch.testing.assert_close(loaded_weight, weight[perm])
torch.testing.assert_close(_load(scale, perm, dim=0), scale[perm])
def test_permuted_wo_a_consumes_fused_output():
"""Permuting wo_a's input columns lets it read the kernel's O layout."""
torch.manual_seed(1)
out_features, num_tokens = 32, 4
in_features = WV_GROUP_SIZE * HEAD_DIM
weight = torch.randn(out_features, in_features)
scale = torch.randint(0, 256, (out_features, in_features // 32), dtype=torch.uint8)
standard_o = torch.randn(num_tokens, in_features)
expected = standard_o @ weight.T
# The kernel emits the same values in the fused chunk order.
perm = o_fused_permutation(WV_GROUP_SIZE, HEAD_DIM)
fused_o = standard_o[:, perm]
loaded_weight = _load(weight, perm, dim=1)
torch.testing.assert_close(loaded_weight, weight[:, perm])
chunk_perm = o_fused_chunk_permutation(WV_GROUP_SIZE, HEAD_DIM)
torch.testing.assert_close(_load(scale, chunk_perm, dim=1), scale[:, chunk_perm])
# Unlike wq_b, this permutation sits inside the 4096-term reduction, so the
# summation order changes and the result differs in the last few ulps.
torch.testing.assert_close(
fused_o @ loaded_weight.T, expected, rtol=1e-3, atol=1e-3
)
File diff suppressed because it is too large Load Diff
-524
View File
@@ -1,524 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Integration tests for FlexAttention backend vs default backend."""
import pytest
import torch
from packaging import version
from tests.utils import set_random_seed
from tests.v1.attention.utils import (
BatchSpec,
create_common_attn_metadata,
create_standard_kv_cache_spec,
create_vllm_config,
)
from vllm.config import AttentionConfig
from vllm.model_executor.layers.attention import Attention
from vllm.v1.attention.backends.flex_attention import (
BlockSparsityHint,
FlexAttentionMetadataBuilder,
physical_to_logical_mapping,
)
from ..models.utils import check_embeddings_close, check_logprobs_close
TORCH_VERSION = version.parse(torch.__version__)
MINIMUM_TORCH_VERSION = version.parse("2.7.0")
DIRECT_BUILD_VERSION = version.parse("2.9.dev0")
@pytest.mark.parametrize(
("supports_small_blocks", "uses_paged_kv", "expected"),
[
(True, True, (16, 16)),
(True, False, (128, 128)),
(False, True, (128, 128)),
(False, False, (128, 128)),
],
)
def test_flex_attention_default_block_sizes(
supports_small_blocks: bool,
uses_paged_kv: bool,
expected: tuple[int, int],
):
block_sizes = FlexAttentionMetadataBuilder._get_block_sizes(
AttentionConfig(),
supports_small_blocks=supports_small_blocks,
cache_block_size=16,
uses_paged_kv=uses_paged_kv,
)
assert block_sizes == expected
def test_flex_attention_explicit_block_sizes_override_encoder_defaults():
block_sizes = FlexAttentionMetadataBuilder._get_block_sizes(
AttentionConfig(
flex_attn_q_block_size=64,
flex_attn_kv_block_size=32,
),
supports_small_blocks=True,
cache_block_size=16,
uses_paged_kv=False,
)
assert block_sizes == (64, 32)
@pytest.mark.skipif(
not torch.cuda.is_available() or TORCH_VERSION < MINIMUM_TORCH_VERSION,
reason="CUDA not available or PyTorch version < 2.7",
)
def test_flex_attention_full_cudagraphs(vllm_runner):
"""Test the numerics for flex attention full cudagraphs support."""
model_name = "Qwen/Qwen2.5-1.5B-Instruct"
seed = 42
max_tokens = 24
num_logprobs = 5
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
]
# Run with flex attention eager
set_random_seed(seed)
with vllm_runner(
model_name,
runner="generate",
tensor_parallel_size=1,
num_gpu_blocks_override=128,
enforce_eager=True,
attention_config={"backend": "FLEX_ATTENTION"},
) as llm_flex:
output_eager = llm_flex.generate_greedy_logprobs(
prompts, max_tokens, num_logprobs
)
# Run with flex attention compiled
set_random_seed(seed)
with vllm_runner(
model_name,
runner="generate",
tensor_parallel_size=1,
num_gpu_blocks_override=128,
enforce_eager=False,
gpu_memory_utilization=0.85,
attention_config={"backend": "FLEX_ATTENTION"},
) as llm_default:
output_compile = llm_default.generate_greedy_logprobs(
prompts, max_tokens, num_logprobs
)
check_logprobs_close(
outputs_0_lst=output_eager,
outputs_1_lst=output_compile,
name_0="eager",
name_1="compile",
)
def windowed_causal_mask_mod(b, h, q_idx, kv_idx):
return (kv_idx <= q_idx) & (q_idx - kv_idx < 4)
@pytest.mark.skipif(
not torch.cuda.is_available() or TORCH_VERSION < MINIMUM_TORCH_VERSION,
reason="CUDA not available or PyTorch version < 2.7",
)
def test_flex_attention_custom_mask_full_cudagraphs(vllm_runner, monkeypatch):
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
monkeypatch.setattr(
Attention,
"logical_mask_mod",
staticmethod(windowed_causal_mask_mod),
raising=False,
)
model_name = "Qwen/Qwen2.5-1.5B-Instruct"
seed = 42
max_tokens = 24
num_logprobs = 5
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
]
set_random_seed(seed)
with vllm_runner(
model_name,
runner="generate",
tensor_parallel_size=1,
num_gpu_blocks_override=128,
enforce_eager=True,
attention_config={"backend": "FLEX_ATTENTION"},
) as llm_eager:
output_eager = llm_eager.generate_greedy_logprobs(
prompts, max_tokens, num_logprobs
)
set_random_seed(seed)
with vllm_runner(
model_name,
runner="generate",
tensor_parallel_size=1,
num_gpu_blocks_override=128,
enforce_eager=False,
gpu_memory_utilization=0.85,
compilation_config={
"cudagraph_mode": "FULL",
"cudagraph_capture_sizes": [4],
},
attention_config={"backend": "FLEX_ATTENTION"},
) as llm_cudagraph:
output_cudagraph = llm_cudagraph.generate_greedy_logprobs(
prompts, max_tokens, num_logprobs
)
check_logprobs_close(
outputs_0_lst=output_eager,
outputs_1_lst=output_cudagraph,
name_0="eager",
name_1="cudagraph",
)
@pytest.mark.skipif(
not torch.cuda.is_available() or TORCH_VERSION < MINIMUM_TORCH_VERSION,
reason="CUDA not available or PyTorch version < 2.7",
)
def test_flex_attention_vs_default_backend(vllm_runner):
"""Test that FlexAttention produces the same outputs as the default backend.
This test compares the outputs from the FlexAttention backend with
the default backend, ensuring they are similar when using the same seed.
"""
model_name = "Qwen/Qwen2.5-1.5B-Instruct"
seed = 42
max_tokens = 24
num_logprobs = 5
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
]
# Run with flex attention
set_random_seed(seed)
with vllm_runner(
model_name,
runner="generate",
tensor_parallel_size=1,
num_gpu_blocks_override=128,
enforce_eager=True,
attention_config={"backend": "FLEX_ATTENTION"},
) as llm_flex:
output_flex = llm_flex.generate_greedy_logprobs(
prompts, max_tokens, num_logprobs
)
# Run with default backend
set_random_seed(seed)
with vllm_runner(
model_name,
runner="generate",
tensor_parallel_size=1,
num_gpu_blocks_override=128,
enforce_eager=True,
gpu_memory_utilization=0.85,
) as llm_default:
output_default = llm_default.generate_greedy_logprobs(
prompts, max_tokens, num_logprobs
)
check_logprobs_close(
outputs_0_lst=output_flex,
outputs_1_lst=output_default,
name_0="flex",
name_1="default",
)
@pytest.mark.skipif(
not torch.cuda.is_available() or TORCH_VERSION < MINIMUM_TORCH_VERSION,
reason="CUDA not available or PyTorch version < 2.7",
)
def test_encoder_flex_attention_vs_default_backend(vllm_runner):
"""Test that FlexAttention produces the same outputs as the default backend.
This test compares the outputs from the FlexAttention backend with
the default backend for encoder models.
"""
model_name = "BAAI/bge-base-en-v1.5"
# Exercise packed sequence boundaries inside 128-token FlexAttention
# blocks, including sequences that span more than one block.
prompts = [
"hello " * 120,
"world " * 130,
"attention " * 254,
]
# Run with flex attention
with vllm_runner(
model_name,
runner="pooling",
dtype=torch.bfloat16,
tensor_parallel_size=1,
max_model_len=384,
enforce_eager=True,
attention_config={"backend": "FLEX_ATTENTION"},
) as llm_flex:
flex_outputs = llm_flex.embed(prompts)
# Run with default backend
with vllm_runner(
model_name,
runner="pooling",
dtype=torch.bfloat16,
tensor_parallel_size=1,
max_model_len=384,
enforce_eager=True,
) as llm_default:
default_outputs = llm_default.embed(prompts)
check_embeddings_close(
embeddings_0_lst=flex_outputs,
embeddings_1_lst=default_outputs,
name_0="flex",
name_1="default",
tol=1e-2,
)
@pytest.mark.skipif(
not torch.cuda.is_available() or TORCH_VERSION < DIRECT_BUILD_VERSION,
reason="CUDA not available or PyTorch version < 2.7",
)
def test_block_mask_direct_vs_slow_path():
"""Test that direct path block mask is a superset of slow path.
The direct path may include extra blocks for performance (over-estimation),
but must include all blocks that the slow path determines are necessary.
"""
device = torch.device("cuda")
vllm_config = create_vllm_config(
model_name="Qwen/Qwen2.5-1.5B-Instruct", block_size=16, max_model_len=1024
)
kv_cache_spec = create_standard_kv_cache_spec(vllm_config)
# Use a mixed batch that will create groups spanning multiple sequences
batch_spec = BatchSpec(
seq_lens=[35, 64, 128, 256], query_lens=[33, 5, 32, 64], name="test_mixed_batch"
)
common_attn_metadata = create_common_attn_metadata(
batch_spec, vllm_config.cache_config.block_size, device
)
builder = FlexAttentionMetadataBuilder(kv_cache_spec, [], vllm_config, device)
metadata_direct = builder.build(
common_prefix_len=0, common_attn_metadata=common_attn_metadata
)
builder.direct_build = False
metadata_slow = builder.build(
common_prefix_len=0, common_attn_metadata=common_attn_metadata
)
assert metadata_direct.block_mask is not None
assert metadata_slow.block_mask is not None
# Extract block indices for comparison, B, H are the same
direct_indices = metadata_direct.block_mask.kv_indices[0, 0]
slow_indices = metadata_slow.block_mask.kv_indices[0, 0]
direct_num = metadata_direct.block_mask.kv_num_blocks[0, 0]
slow_num = metadata_slow.block_mask.kv_num_blocks[0, 0]
# main test: every block needed by slow path must be in direct path
num_groups = direct_num.shape[0]
all_contained = True
missing_details = []
for group_idx in range(num_groups):
direct_blocks = set(direct_indices[group_idx, : direct_num[group_idx]].tolist())
slow_blocks = set(slow_indices[group_idx, : slow_num[group_idx]].tolist())
missing_blocks = slow_blocks - direct_blocks
if missing_blocks:
all_contained = False
missing_details.append(
f"Group {group_idx}: missing {sorted(missing_blocks)}"
)
assert all_contained, (
"Direct path is missing blocks required by slow path:\n"
+ "\n".join(missing_details)
)
@pytest.mark.skipif(
not torch.cuda.is_available() or TORCH_VERSION < DIRECT_BUILD_VERSION,
reason="CUDA not available or PyTorch version < 2.9",
)
@pytest.mark.parametrize("direct_build", [True, False])
@torch.inference_mode()
def test_flex_attention_request_count_changes_reuse_compiled_graph(direct_build):
"""Request-count changes preserve attention results and compiled graph reuse."""
from torch._dynamo.testing import CompileCounterWithBackend
from torch.nn.attention.flex_attention import flex_attention
from vllm.v1.attention.backends.flex_attention import get_kernel_options
torch.manual_seed(42)
device = torch.device("cuda")
config = create_vllm_config(
model_name="Qwen/Qwen3-0.6B",
max_model_len=128,
num_gpu_blocks=40,
max_num_seqs=8,
max_num_batched_tokens=128,
)
builder = FlexAttentionMetadataBuilder(
create_standard_kv_cache_spec(config), [], config, device
)
builder.direct_build = direct_build
query = torch.randn(1, 2, 32, 64, device=device)
key = torch.randn(1, 2, 640, 64, device=device)
value = torch.randn_like(key)
kernel_options = get_kernel_options(query, 16, 16, direct_build)
counter = CompileCounterWithBackend("inductor")
compiled_attention = torch.compile(flex_attention, backend=counter, fullgraph=True)
def build_metadata(num_reqs):
# Keep Q/K tensor and block-mask sizes fixed to isolate request counts.
common = create_common_attn_metadata(
BatchSpec(seq_lens=[64] * num_reqs, query_lens=[32 // num_reqs] * num_reqs),
16,
device,
max_block_idx=40,
)
common.block_table_tensor.copy_(
torch.arange(1, num_reqs * 4 + 1, device=device).view(num_reqs, 4)
)
return builder.build(0, common)
def attend(metadata):
return compiled_attention(
query,
key,
value,
block_mask=metadata.block_mask,
kernel_options=kernel_options,
)
def reference(num_reqs):
query_len = 32 // num_reqs
q_idx = torch.arange(32, device=device)[:, None]
kv_idx = torch.arange(640, device=device)[None, :]
request = q_idx // query_len
logical_q = q_idx % query_len + 64 - query_len
logical_kv = kv_idx - (request * 4 + 1) * 16
mask = (logical_kv >= 0) & (logical_kv <= logical_q)
return torch.nn.functional.scaled_dot_product_attention(
query, key, value, attn_mask=mask
)
for num_reqs in (4, 2, 1, 8, 1):
metadata = build_metadata(num_reqs)
torch.testing.assert_close(
attend(metadata), reference(num_reqs), atol=1e-4, rtol=1e-4
)
assert counter.frame_count == 1
if direct_build:
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
output = attend(metadata)
for num_reqs in (8, 2, 1):
metadata = build_metadata(num_reqs)
graph.replay()
torch.testing.assert_close(
output, reference(num_reqs), atol=1e-4, rtol=1e-4
)
def test_physical_to_logical_mapping_handles_reused_blocks():
"""Regression test: reused physical blocks map to the latest logical block.
For sliding-window / hybrid attention layers, physical KV-cache blocks can be
reused over time. The inverse mapping must therefore select the latest
logical block index for a physical block id.
"""
# Padding should not make physical block 0 look live.
block_table = torch.tensor([[6, 0, 0, 0]], dtype=torch.int32)
seq_lens = torch.tensor([1 * 16], dtype=torch.int32) # only 1 block valid
out = physical_to_logical_mapping(
block_table=block_table, seq_lens=seq_lens, block_size=16, total_blocks=10
)
assert out[0, 0].item() == -1
assert out[0, 6].item() == 0
# If a physical block id appears multiple times (block reuse), mapping should
# point to the latest logical block index.
block_table2 = torch.tensor([[2, 2, 5]], dtype=torch.int32)
seq_lens2 = torch.tensor([3 * 16], dtype=torch.int32)
out2 = physical_to_logical_mapping(
block_table=block_table2, seq_lens=seq_lens2, block_size=16, total_blocks=8
)
assert out2[0, 2].item() == 1
@pytest.mark.skipif(
not torch.cuda.is_available() or TORCH_VERSION < DIRECT_BUILD_VERSION,
reason="CUDA not available or PyTorch version < 2.9",
)
def test_block_sparsity_hint_prunes_blocks():
"""Test that BlockSparsityHint prunes KV blocks from the direct build path.
Uses a hint that only keeps the diagonal (q_block == kv_block) to verify
that off-diagonal blocks are excluded from the resulting BlockMask.
"""
device = torch.device("cuda")
vllm_config = create_vllm_config(
model_name="facebook/opt-125m",
block_size=16,
max_model_len=1024,
)
kv_cache_spec = create_standard_kv_cache_spec(vllm_config)
batch_spec = BatchSpec(
seq_lens=[256],
query_lens=[256],
name="test_sparsity_hint",
)
common_attn_metadata = create_common_attn_metadata(
batch_spec, vllm_config.cache_config.block_size, device
)
builder = FlexAttentionMetadataBuilder(kv_cache_spec, [], vllm_config, device)
metadata_no_hint = builder.build(
common_prefix_len=0, common_attn_metadata=common_attn_metadata
)
metadata_no_hint.block_mask = metadata_no_hint._build_block_mask_direct()
assert metadata_no_hint.block_mask.kv_num_blocks.max().item() > 1
def diagonal_hint(q_block_idx, kv_block_idx, block_size):
return q_block_idx == kv_block_idx
metadata_with_hint = builder.build(
common_prefix_len=0, common_attn_metadata=common_attn_metadata
)
metadata_with_hint.block_sparsity_hint = BlockSparsityHint(
hint_fn=diagonal_hint,
)
metadata_with_hint.block_mask = metadata_with_hint._build_block_mask_direct()
assert metadata_with_hint.block_mask.kv_num_blocks.max().item() <= 1
if __name__ == "__main__":
pytest.main([__file__])
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -1,620 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit test for fused_indexer_q_rope_quant.
Compares the fused Triton kernel against the unfused reference flow used by
the DeepseekV4 indexer in model_tracking:
q_rot = ops.rotary_embedding(positions, q, None, head_dim, cos_sin_cache,
is_neox_style=False,
rope_dim_offset=head_dim - rope_dim)
q_fp8, q_scale = per_token_group_quant_fp8(q_rot, head_dim, use_ue8m0=True)
weights_out = weights * q_scale * softmax_scale * head_scale
Expects bit-exact equality on both q_fp8 and weights_out.
"""
import contextlib
import importlib
from unittest import mock
import pytest
import torch
from vllm import _custom_ops as ops
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
per_token_group_quant_fp8,
)
from vllm.models.deepseek_v4.common.ops import fused_indexer_q_rope_quant
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
from vllm.utils.import_utils import has_cutedsl
HEAD_DIM = 128
ROPE_DIM = 64
MAX_POS = 4096
CUTEDSL_MODULE = "vllm.models.deepseek_v4.nvidia.ops.fused_indexer_q_cutedsl"
def quantize_to_mxfp4(
x: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Reference MXFP4 quantization.
Args:
x: [..., head_dim] where head_dim is divisible by 32
Returns:
packed: [..., head_dim//2] uint8 2 E2M1 nibbles/byte, low nibble = even index
scales: [..., head_dim//32] uint8 1 ue8m0 byte
"""
MXFP4_BLOCK_SIZE = 32
orig_shape = x.shape
head_dim = orig_shape[-1]
n_blocks = head_dim // MXFP4_BLOCK_SIZE
x_f32 = x.float().reshape(-1, n_blocks, MXFP4_BLOCK_SIZE)
# Per-block ue8m0 scale: 2^ceil(log2(amax / 6.0)), stored as byte = exp + 127
# 6 * 2^-126 is from https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/blob/main/inference/kernel.py#L163
amax = x_f32.abs().amax(dim=-1, keepdim=True).clamp(min=6 * (2**-126))
log2_ratio = (amax * (1.0 / 6.0)).log2().ceil().clamp(-127.0, 127.0)
scale = log2_ratio.exp2()
ue8m0 = (log2_ratio + 127.0).to(torch.uint8) # [*, n_blocks]
# E2M1 round-to-nearest-even: midpoints round to the even code.
# E2M1 values: [0.00, 0.50, 1.00, 1.50, 2.00, 3.00, 4.00, 6.00]
# boundaries: [ 0.25, 0.75, 1.25, 1.75, 2.50, 3.50, 5.00]
x_scaled = (x_f32 / scale).clamp(-6.0, 6.0)
abs_x = x_scaled.abs()
code = torch.zeros_like(abs_x, dtype=torch.int32)
code = torch.where(abs_x > 0.25, 1, code)
code = torch.where(abs_x >= 0.75, 2, code)
code = torch.where(abs_x > 1.25, 3, code)
code = torch.where(abs_x >= 1.75, 4, code)
code = torch.where(abs_x > 2.5, 5, code)
code = torch.where(abs_x >= 3.5, 6, code)
code = torch.where(abs_x > 5.0, 7, code)
sign = ((x_scaled.view(torch.int32) >> 31) & 1).to(torch.uint8)
nibble = code.to(torch.uint8) | (sign << 3)
# Pack: even-index element → low nibble, odd-index → high nibble
nibble_flat = nibble.reshape(-1, head_dim)
packed = (nibble_flat[:, 0::2] | (nibble_flat[:, 1::2] << 4)).contiguous()
packed = packed.reshape(*orig_shape[:-1], head_dim // 2)
scales = ue8m0.view(*orig_shape[:-1], n_blocks)
return packed, scales
def _reference(
positions: torch.Tensor,
q: torch.Tensor,
cos_sin_cache: torch.Tensor,
weights: torch.Tensor,
softmax_scale: float,
head_scale: float,
n_head: int,
use_fp4: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
q_rot = q.clone()
ops.rotary_embedding(
positions,
q_rot,
None,
HEAD_DIM,
cos_sin_cache,
False, # is_neox_style=False → GPT-J interleaved
HEAD_DIM - ROPE_DIM, # rope_dim_offset → rotate the tail
False,
)
if use_fp4:
q_packed, ue8m0 = quantize_to_mxfp4(q_rot.view(-1, n_head, HEAD_DIM))
# Pack 4 ue8m0 bytes into 1 int32
q_scale = ue8m0.view(torch.int32).squeeze(-1)
# FP4 path: q_scale stays separate (cannot be folded into a per-token scalar)
weights_out = weights.to(torch.float32) * softmax_scale * head_scale
return (q_packed, q_scale), weights_out
else:
q_fp8, q_scale = per_token_group_quant_fp8(
q_rot.view(-1, HEAD_DIM).contiguous(),
HEAD_DIM,
use_ue8m0=True,
)
q_fp8 = q_fp8.view(-1, n_head, HEAD_DIM)
q_scale = q_scale.view(-1, n_head)
weights_out = weights.to(torch.float32) * q_scale * softmax_scale * head_scale
return q_fp8, weights_out
@pytest.mark.parametrize("num_tokens", [1, 7, 32, 257, 1023])
@pytest.mark.parametrize("cache_dtype", [torch.float32, torch.bfloat16])
@pytest.mark.parametrize(
"use_fp4",
[
False,
pytest.param(
True,
marks=pytest.mark.skipif(
not (
current_platform.is_cuda()
and current_platform.is_device_capability_family(100)
),
reason="MXFP4 indexer cache requires an SM100-family GPU",
),
),
],
)
@pytest.mark.parametrize("use_cutedsl", [False, True])
@pytest.mark.parametrize("n_head", [32, 64])
@torch.inference_mode()
def test_fused_indexer_q_rope_quant_matches_unfused(
num_tokens, cache_dtype, use_fp4, use_cutedsl, n_head
):
if use_cutedsl and not has_cutedsl():
pytest.skip("cutedsl (cutlass) not installed")
device = "cuda"
torch.manual_seed(0)
q = torch.randn(num_tokens, n_head, HEAD_DIM, dtype=torch.bfloat16, device=device)
positions = torch.randint(
0, MAX_POS, (num_tokens,), dtype=torch.int64, device=device
)
cos_sin_cache = torch.randn(MAX_POS, ROPE_DIM, dtype=cache_dtype, device=device)
weights = torch.randn(num_tokens, n_head, dtype=torch.bfloat16, device=device)
softmax_scale = HEAD_DIM**-0.5
# head_scale must be an exact power of two for bit-exactness to be a fair
# expectation: the triton kernel multiplies by softmax_scale and head_scale
# separately (two roundings) while the cutedsl kernels fold them into one
# fp32 scalar first (one rounding). Those agree only when one factor is a
# pow2. n_head**-0.5 is a pow2 at 64 heads but not at 32.
head_scale = 0.125
q_quant_ref, weights_ref = _reference(
positions,
q,
cos_sin_cache,
weights,
softmax_scale,
head_scale,
n_head,
use_fp4,
)
# use_cutedsl=False: force the triton path even when cutedsl is installed
# by patching the dispatcher's has_cutedsl() binding to return False.
cutedsl_patch = (
mock.patch(
"vllm.models.deepseek_v4.common.ops.fused_indexer_q.has_cutedsl",
return_value=False,
)
if not use_cutedsl
else contextlib.nullcontext()
)
with cutedsl_patch:
q_quant_fused, weights_fused = fused_indexer_q_rope_quant(
positions,
q.clone(),
cos_sin_cache,
weights,
softmax_scale,
head_scale,
use_fp4,
)
if use_fp4:
q_quant_ref, q_scale_ref = q_quant_ref
q_quant_fused, q_scale_fused = q_quant_fused
assert torch.equal(q_scale_ref, q_scale_fused), (
f"q_scale mismatch: "
f"{(q_scale_ref != q_scale_fused).sum().item()} "
f"/ {q_scale_ref.numel()} bytes differ"
)
# fp8 tensors aren't directly comparable via torch.equal — reinterpret as int8.
ref_bits = q_quant_ref.view(torch.int8)
fused_bits = q_quant_fused.view(torch.int8)
assert torch.equal(ref_bits, fused_bits), (
f"q_quant_fused mismatch: "
f"{(ref_bits != fused_bits).sum().item()} / {ref_bits.numel()} bytes differ"
)
assert weights_fused.dtype == torch.float32
assert torch.equal(weights_ref, weights_fused), (
f"weights mismatch: max abs diff "
f"{(weights_ref - weights_fused).abs().max().item()}"
)
@pytest.mark.skipif(not has_cutedsl(), reason="cutedsl (cutlass) not installed")
@pytest.mark.parametrize(
"use_fp4",
[
False,
pytest.param(
True,
marks=pytest.mark.skipif(
not (
current_platform.is_cuda()
and current_platform.is_device_capability_family(100)
),
reason="MXFP4 indexer cache requires an SM100-family GPU",
),
),
],
)
@pytest.mark.parametrize("n_head", [32, 64])
@torch.inference_mode()
def test_cutedsl_indexer_q_writes_stay_within_num_tokens(use_fp4, n_head):
"""The cutedsl kernels must not store past ``num_tokens``.
The launcher rounds ``num_tokens * threads_per_token`` up to whole 128-thread
blocks, so the trailing block is only partially in range unless
``threads_per_token`` is itself a multiple of the block size. With 32 heads
and coarsen=4 (picked for >= 512 tokens) a token needs 64 threads, so an odd
token count leaves half of the last block addressing token ``num_tokens``.
Every output gets one extra sentinel row that the kernel must leave alone.
"""
mod = importlib.import_module(CUTEDSL_MODULE)
device = "cuda"
torch.manual_seed(0)
num_tokens = 935
sentinel_u8, sentinel_f32 = 0x55, -12345.0
# positions/q are padded too, so an out-of-bounds *load* stays addressable
# and the assertions below are about stores only.
positions = torch.zeros(num_tokens + 1, dtype=torch.int64, device=device)
q = torch.randn(
num_tokens + 1, n_head, HEAD_DIM, dtype=torch.bfloat16, device=device
)
cos_sin_cache = torch.randn(MAX_POS, ROPE_DIM, dtype=torch.float32, device=device)
weights = torch.randn(num_tokens + 1, n_head, dtype=torch.bfloat16, device=device)
weights_out = torch.full(
(num_tokens + 1, n_head), sentinel_f32, dtype=torch.float32, device=device
)
outputs = {"weights_out": weights_out}
if use_fp4:
outputs["q_packed"] = torch.full(
(num_tokens + 1, n_head, HEAD_DIM // 2),
sentinel_u8,
dtype=torch.uint8,
device=device,
)
outputs["q_scale"] = torch.full(
(num_tokens + 1, n_head, HEAD_DIM // 32),
sentinel_u8,
dtype=torch.uint8,
device=device,
)
mod._INDEXER_Q_MXFP4_KERNEL(
positions=positions[:num_tokens],
q=q[:num_tokens],
cos_sin_cache=cos_sin_cache,
weights=weights[:num_tokens],
weights_softmax_scale=1.0,
weights_head_scale=1.0,
q_packed=outputs["q_packed"][:num_tokens],
q_scale=outputs["q_scale"][:num_tokens],
weights_out=weights_out[:num_tokens],
)
else:
outputs["q_fp8"] = torch.full(
(num_tokens + 1, n_head, HEAD_DIM),
sentinel_u8,
dtype=torch.uint8,
device=device,
)
mod._INDEXER_Q_FP8_KERNEL(
positions=positions[:num_tokens],
q=q[:num_tokens],
cos_sin_cache=cos_sin_cache,
weights=weights[:num_tokens],
weights_softmax_scale=1.0,
weights_head_scale=1.0,
q_fp8=outputs["q_fp8"][:num_tokens].view(torch.float8_e4m3fn),
weights_out=weights_out[:num_tokens],
)
torch.accelerator.synchronize()
clobbered = {}
for name, tensor in outputs.items():
sentinel = sentinel_f32 if tensor.dtype == torch.float32 else sentinel_u8
n_changed = (tensor[num_tokens] != sentinel).sum().item()
if n_changed:
clobbered[name] = n_changed
assert not clobbered, (
f"kernel wrote past token {num_tokens - 1} into the sentinel row: {clobbered}"
)
def _indexer_k_reference(
k_pre, positions, cos_sin_cache, rms_norm_weight, compress_ratio, use_fp4
):
k = k_pre.float()
k = k * torch.rsqrt(k.square().mean(dim=-1, keepdim=True) + 1e-20)
k = (k * rms_norm_weight.float()).to(torch.bfloat16).float()
group_positions = positions // compress_ratio * compress_ratio
cos, sin = cos_sin_cache[group_positions].float().chunk(2, dim=-1)
even, odd = k[:, 64::2], k[:, 65::2]
rotated = torch.stack((even * cos - odd * sin, odd * cos + even * sin), dim=-1)
k = torch.cat((k[:, :64], rotated.flatten(1)), dim=-1).to(torch.bfloat16)
if use_fp4:
return quantize_to_mxfp4(k)
exponent = k.float().abs().amax(dim=-1, keepdim=True).clamp(min=1e-4) / 448
scale = exponent.log2().ceil().exp2()
values = (k.float() / scale).clamp(-448, 448).to(torch.float8_e4m3fn)
if mqa_logits_use_mx_scales():
# The power-of-two scale's exponent as 4 equal UE8M0 bytes.
ue8m0 = (scale.view(torch.int32) >> 23).to(torch.uint8)
return values.view(torch.uint8), ue8m0.repeat(1, 4)
return values.view(torch.uint8), scale.view(torch.uint8)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA kernel")
@pytest.mark.parametrize("num_tokens", [1, 17, 257])
@pytest.mark.parametrize("compress_ratio", [1, 2])
@pytest.mark.parametrize("use_fp4", [False, True])
@pytest.mark.parametrize("cache_dtype", [torch.float32, torch.bfloat16])
@torch.inference_mode()
def test_indexer_k_inserts_only_valid_groups_into_padded_pages(
num_tokens, compress_ratio, use_fp4, cache_dtype
):
"""Insert group ends and preserve skipped slots, graph rows, and page padding."""
from vllm.models.deepseek_v41.common.ops.indexer_k_store import (
indexer_k_norm_rope_store,
)
if use_fp4 and not current_platform.is_device_capability_family(100):
pytest.skip("MXFP4 indexer cache requires an SM100-family GPU")
torch.manual_seed(17)
device = "cuda"
k_pre = torch.randn(num_tokens + 3, 128, device=device, dtype=torch.bfloat16)
norm_weight = torch.randn(128, device=device, dtype=torch.bfloat16)
positions = torch.arange(7, num_tokens + 10, device=device)
slots = torch.randperm(2 * num_tokens, device=device)[:num_tokens].clone()
slots[5::7] = -1
valid = (slots >= 0) & ((positions[:num_tokens] + 1) % compress_ratio == 0)
k_pre[3::7] = 0
k_pre[8::11] *= 1e-10
k_pre[:num_tokens][~valid] = float("nan")
k_pre[num_tokens:] = float("nan")
positions[num_tokens:] = MAX_POS + 10
angles = torch.randn(MAX_POS, 32, device=device)
cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1).to(cache_dtype)
block_size = 16
value_bytes = 64 if use_fp4 else 128
page_bytes = block_size * (value_bytes + 4)
num_blocks = (2 * num_tokens + block_size - 1) // block_size + 1
storage = torch.full(
(num_blocks, page_bytes + 128), 0xA5, device=device, dtype=torch.uint8
)
cache = storage.as_strided(
(num_blocks, block_size, value_bytes + 4),
(storage.stride(0), value_bytes + 4, 1),
)
expected = storage.clone()
values, scales = _indexer_k_reference(
k_pre[:num_tokens][valid],
positions[:num_tokens][valid],
cos_sin,
norm_weight,
compress_ratio,
use_fp4,
)
block_ids = slots[valid] // block_size
block_offsets = slots[valid] % block_size
value_offsets = block_offsets[:, None] * value_bytes + torch.arange(
value_bytes, device=device
)
scale_offsets = block_size * value_bytes + block_offsets[:, None] * 4
scale_offsets = scale_offsets + torch.arange(4, device=device)
expected[block_ids[:, None], value_offsets] = values
expected[block_ids[:, None], scale_offsets] = scales
indexer_k_norm_rope_store(
k_pre,
positions,
cos_sin,
norm_weight,
1e-20,
cache,
slots,
compress_ratio,
use_fp4,
)
# Fused RoPE can change zero signs without changing their numerical values.
actual_values = storage[block_ids[:, None], value_offsets]
if use_fp4:
for shift in (0, 4):
zero_mask = (actual_values >> shift) & 7 == 0
actual_values &= ~(zero_mask.to(torch.uint8) << (shift + 3))
zero_mask = (values >> shift) & 7 == 0
values &= ~(zero_mask.to(torch.uint8) << (shift + 3))
else:
actual_values[actual_values == 128] = 0
values[values == 128] = 0
storage[block_ids[:, None], value_offsets] = actual_values
expected[block_ids[:, None], value_offsets] = values
torch.testing.assert_close(storage, expected, rtol=0, atol=0)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA kernel")
@pytest.mark.parametrize("use_fp4", [False, True])
@torch.inference_mode()
def test_indexer_k_cuda_graph_replay_reads_current_projection(use_fp4):
"""Replay must consume updated keys and slot mapping through the captured API."""
from vllm.models.deepseek_v41.common.ops.indexer_k_store import (
indexer_k_norm_rope_store,
)
if use_fp4 and not current_platform.is_device_capability_family(100):
pytest.skip("MXFP4 indexer cache requires an SM100-family GPU")
torch.manual_seed(18)
k_pre = torch.randn(19, 128, device="cuda", dtype=torch.bfloat16)
norm_weight = torch.ones(128, device="cuda", dtype=torch.bfloat16)
positions = torch.arange(19, device="cuda")
cos_sin = torch.zeros(32, 64, device="cuda")
cos_sin[:, :32] = 1
slots = torch.arange(17, device="cuda")
cache = torch.full(
(2, 16, (64 if use_fp4 else 128) + 4),
0xA5,
device="cuda",
dtype=torch.uint8,
)
def run():
indexer_k_norm_rope_store(
k_pre,
positions,
cos_sin,
norm_weight,
1e-20,
cache,
slots,
2,
use_fp4,
)
run()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
run()
k_pre.neg_()
slots.add_(4)
cache.fill_(0xA5)
graph.replay()
replayed = cache.clone()
cache.fill_(0xA5)
run()
torch.testing.assert_close(cache, replayed, rtol=0, atol=0)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm cache layout")
@pytest.mark.parametrize("block_size", [16, 64, 128])
@pytest.mark.parametrize("compress_ratio", [1, 2])
@torch.inference_mode()
def test_indexer_k_store_roundtrips_through_rocm_gather(block_size, compress_ratio):
"""ROCm reads the indexer K cache back with the 16x16-tiled value layout.
``cp_gather_indexer_k_quant_cache_triton`` (prefill) and aiter's
``Preshuffle=True`` paged kernel (decode) both pick that layout whenever
``block_size > 1``, so a row-major store would feed the indexer permuted
key bytes and randomize its top-k. Assert the write/read pair is exact.
"""
from vllm.models.deepseek_v41.common.ops.indexer_k_store import (
indexer_k_norm_rope_store,
)
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
cp_gather_indexer_k_quant_cache_triton,
)
torch.manual_seed(0)
device = "cuda"
num_tokens = 3 * block_size + 5
num_blocks = num_tokens // block_size + 2
k_pre = torch.randn(num_tokens, HEAD_DIM, device=device, dtype=torch.bfloat16)
norm_weight = torch.randn(HEAD_DIM, device=device, dtype=torch.bfloat16)
positions = torch.arange(num_tokens, device=device)
angles = torch.randn(MAX_POS, ROPE_DIM // 2, device=device)
cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1).float()
cache = torch.zeros(
num_blocks, block_size, HEAD_DIM + 4, device=device, dtype=torch.uint8
)
slots = torch.arange(num_tokens, device=device)
indexer_k_norm_rope_store(
k_pre,
positions,
cos_sin,
norm_weight,
1e-20,
cache,
slots,
compress_ratio,
False,
)
k_fp8 = torch.zeros(num_tokens, HEAD_DIM, device=device, dtype=torch.float8_e4m3fn)
k_scale = torch.zeros(num_tokens, 4, device=device, dtype=torch.uint8)
cp_gather_indexer_k_quant_cache_triton(
cache,
k_fp8,
k_scale,
torch.arange(num_blocks, device=device, dtype=torch.int32).view(1, -1),
torch.tensor([0, num_tokens], device=device, dtype=torch.int32),
torch.zeros(num_tokens, device=device, dtype=torch.int32),
)
expected_values, expected_scale = _indexer_k_reference(
k_pre, positions, cos_sin, norm_weight, compress_ratio, False
)
emitted = (positions + 1) % compress_ratio == 0
gathered = k_fp8.view(torch.uint8)
gathered = torch.where(gathered == 128, gathered.new_zeros(()), gathered)
expected_values = torch.where(
expected_values == 128, expected_values.new_zeros(()), expected_values
)
torch.testing.assert_close(
gathered[emitted], expected_values[emitted], rtol=0, atol=0
)
torch.testing.assert_close(
k_scale[emitted], expected_scale[emitted], rtol=0, atol=0
)
@pytest.mark.parametrize("use_cutedsl", [False, True])
@pytest.mark.skipif(
not (
current_platform.is_cuda() and current_platform.is_device_capability_family(100)
),
reason="MXFP4 indexer cache requires an SM100-family GPU",
)
@torch.inference_mode()
def test_fused_indexer_q_rope_quant_writes_bf16_weights(use_cutedsl):
"""The MXFP4 path can emit the per-head weights in bf16 (what DeepGEMM's
sparse MQA-logits kernels take) instead of fp32; the values are the fp32
result rounded once, so the scoring kernel needs no cast."""
if use_cutedsl and not has_cutedsl():
pytest.skip("cutedsl (cutlass) not installed")
device = "cuda"
torch.manual_seed(0)
num_tokens, n_head = 257, 32
q = torch.randn(num_tokens, n_head, HEAD_DIM, dtype=torch.bfloat16, device=device)
positions = torch.randint(
0, MAX_POS, (num_tokens,), dtype=torch.int64, device=device
)
cos_sin_cache = torch.randn(MAX_POS, ROPE_DIM, dtype=torch.float32, device=device)
weights = torch.randn(num_tokens, n_head, dtype=torch.bfloat16, device=device)
softmax_scale, head_scale = HEAD_DIM**-0.5, 0.125 # pow2 head_scale, see above
_, weights_ref = _reference(
positions, q, cos_sin_cache, weights, softmax_scale, head_scale, n_head, True
)
cutedsl_patch = (
mock.patch(
"vllm.models.deepseek_v4.common.ops.fused_indexer_q.has_cutedsl",
return_value=False,
)
if not use_cutedsl
else contextlib.nullcontext()
)
with cutedsl_patch:
_, weights_fused = fused_indexer_q_rope_quant(
positions,
q.clone(),
cos_sin_cache,
weights,
softmax_scale,
head_scale,
use_fp4=True,
weights_out_dtype=torch.bfloat16,
)
assert weights_fused.dtype == torch.bfloat16
assert torch.equal(weights_fused, weights_ref.to(torch.bfloat16))
File diff suppressed because it is too large Load Diff
-310
View File
@@ -1,310 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import torch.distributed as dist
import torch.multiprocessing as mp
from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.distributed.parallel_state import (
init_distributed_environment,
initialize_model_parallel,
)
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
from vllm.utils.system_utils import update_environment_variables
_SHAPES = (
(129, 512),
(129, 768),
(257, 768),
(1023, 4224),
(1024, 4224),
(8191, 4224),
)
_N = 512
def _reference(
x: torch.Tensor,
weight: torch.Tensor,
weight_scale: torch.Tensor | None,
world_size: int,
group: dist.ProcessGroup,
all_reduce: bool,
) -> torch.Tensor:
M = x.shape[0]
padded_M = (M + world_size - 1) // world_size * world_size
partial = torch.empty(
(padded_M, weight.shape[0]),
dtype=x.dtype,
device=x.device,
)
if weight_scale is None:
torch.mm(x, weight.T, out=partial[:M])
else:
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
mxfp8_e4m3_quantize,
)
x_q, x_sf = mxfp8_e4m3_quantize(x, is_sf_swizzled_layout=True)
partial[:M] = torch._scaled_mm(
x_q,
weight.T,
x_sf.view(torch.float8_e8m0fnu),
weight_scale.view(torch.float8_e8m0fnu),
out_dtype=x.dtype,
)
if padded_M > M:
partial[M:].zero_()
if all_reduce:
dist.all_reduce(partial, group=group)
return partial[:M]
output = torch.empty(
(padded_M // world_size, weight.shape[0]),
dtype=x.dtype,
device=x.device,
)
dist.reduce_scatter_single(output, partial, group=group)
return output
def _assert_valid_rows_close(
actual: torch.Tensor,
expected: torch.Tensor,
M: int,
rank: int,
all_reduce: bool,
) -> None:
if all_reduce:
torch.testing.assert_close(actual, expected, rtol=5e-2, atol=4.0)
return
rank_start = rank * actual.shape[0]
valid_rows = min(max(M - rank_start, 0), actual.shape[0])
torch.testing.assert_close(
actual[:valid_rows],
expected[:valid_rows],
rtol=5e-2,
atol=4.0,
)
# Sequence-parallel callers may pass an unpadded M and read the padding
# rows back (they flow through per-token kernels before being dropped),
# so RS must leave them zero like ``sp_reduce_scatter`` does.
assert not actual[valid_rows:].any()
def _run_mode(
*,
all_reduce: bool,
device: torch.device,
group: dist.ProcessGroup,
rank: int,
world_size: int,
weights: dict[int, torch.Tensor],
backend: str,
) -> None:
# cute_dsl is unavailable off CUDA, so import it only inside the GPU worker.
from vllm.model_executor.kernels.linear.cute_dsl.gemm_rs_ar import GemmRsAr
gemm_rs_ar = GemmRsAr(
max_M=max(M for M, _ in _SHAPES),
N=_N,
all_reduce=all_reduce,
)
from vllm.config.quantization import QuantizationConfigArgs
from vllm.model_executor.layers.linear import RowParallelLinear
from vllm.model_executor.layers.quantization.online.base import (
OnlineQuantizationConfig,
)
quant_config = (
None
if backend == "bf16"
else OnlineQuantizationConfig(QuantizationConfigArgs(linear="mxfp8"))
)
projections = {}
operands = {}
for K, w in weights.items():
with torch.device(device):
linear = RowParallelLinear(
K * world_size,
_N,
bias=False,
params_dtype=torch.bfloat16,
quant_config=quant_config,
)
# Select the fused path before online quantization replaces meta weights.
assert gemm_rs_ar.can_run(linear)
linear.weight = torch.nn.Parameter(w.clone(), requires_grad=False)
linear.quant_method.process_weights_after_loading(linear)
projections[K] = linear
weight = linear.weight
if backend == "flashinfer_cutedsl":
weight = weight.t()
operands[K] = (weight, getattr(linear, "weight_scale", None))
input_generator = torch.Generator(device=device)
# Alternating shapes exercise producer-flag reuse across different grids
# and CTA-group choices.
for M, K in (*_SHAPES, *_SHAPES[::-1]):
input_generator.manual_seed(2000 + M + K)
x = torch.randn(
M,
K,
dtype=torch.bfloat16,
device=device,
generator=input_generator,
)
expected = _reference(x, *operands[K], world_size, group, all_reduce)
actual = gemm_rs_ar.apply(x, projections[K])
torch.accelerator.synchronize(device)
_assert_valid_rows_close(actual, expected, M, rank, all_reduce)
if all_reduce:
# AR outputs must remain valid after the symmetric workspace is reused.
lifetime_M, lifetime_K = 257, 768
input_generator.manual_seed(2500)
lifetime_x = torch.randn(
lifetime_M,
lifetime_K,
dtype=torch.bfloat16,
device=device,
generator=input_generator,
)
first_output = gemm_rs_ar.apply(lifetime_x, projections[lifetime_K])
first_snapshot = first_output.clone()
second_output = gemm_rs_ar.apply(-lifetime_x, projections[lifetime_K])
torch.accelerator.synchronize(device)
assert first_output.data_ptr() != second_output.data_ptr()
torch.testing.assert_close(first_output, first_snapshot, rtol=0, atol=0)
graph_M, graph_K = 1025, 4224
input_generator.manual_seed(3000)
graph_x = torch.randn(
graph_M,
graph_K,
dtype=torch.bfloat16,
device=device,
generator=input_generator,
)
graph_expected = _reference(
graph_x,
*operands[graph_K],
world_size,
group,
all_reduce,
)
graph_output = torch.empty_like(graph_expected)
capture_stream = torch.cuda.Stream()
capture_stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(capture_stream):
for _ in range(3):
graph_output.copy_(gemm_rs_ar.apply(graph_x, projections[graph_K]))
capture_stream.synchronize()
dist.barrier(group=group)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=capture_stream):
graph_output.copy_(gemm_rs_ar.apply(graph_x, projections[graph_K]))
torch.cuda.current_stream().wait_stream(capture_stream)
dist.barrier(group=group)
for _ in range(3):
graph.replay()
torch.accelerator.synchronize(device)
_assert_valid_rows_close(
graph_output,
graph_expected,
graph_M,
rank,
all_reduce,
)
dist.barrier(group=group)
del gemm_rs_ar, capture_stream, graph, graph_output
def _worker(local_rank: int, world_size: int, master_port: int, backend: str) -> None:
# This module pulls in cute_dsl, which is unavailable off CUDA, so importing
# it at module level would fail collection. Import it where it is used, as
# `kda.py` does.
device = torch.device("cuda", local_rank)
torch.accelerator.set_device_index(device)
update_environment_variables(
{
"RANK": str(local_rank),
"LOCAL_RANK": str(local_rank),
"WORLD_SIZE": str(world_size),
"MASTER_ADDR": "localhost",
"MASTER_PORT": str(master_port),
}
)
init_distributed_environment()
config = VllmConfig()
config.model_config = ModelConfig(dtype="bfloat16")
config.kernel_config.linear_backend = "auto" if backend == "bf16" else backend
with set_current_vllm_config(config):
initialize_model_parallel(tensor_model_parallel_size=world_size)
group = dist.group.WORLD
rank = dist.get_rank(group)
weight_generator = torch.Generator(device=device)
weights = {}
for K in {K for _, K in _SHAPES}:
weight_generator.manual_seed(1000 + rank * 10 + K)
weights[K] = torch.randn(
_N,
K,
dtype=torch.bfloat16,
device=device,
generator=weight_generator,
)
# Production binds one mode per worker; exercise both mode-bound instances
# sequentially in this test without implying that both are initialized.
with set_current_vllm_config(config):
for all_reduce in (False, True):
_run_mode(
all_reduce=all_reduce,
device=device,
group=group,
rank=rank,
world_size=world_size,
weights=weights,
backend=backend,
)
cleanup_dist_env_and_memory()
@pytest.mark.parametrize(
"backend", ["bf16", "flashinfer_cutlass", "flashinfer_cutedsl"]
)
@pytest.mark.distributed(num_gpus=2)
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="GEMM-RS/AR requires SM100",
)
def test_gemm_rs_ar(monkeypatch: pytest.MonkeyPatch, backend: str) -> None:
world_size = 2
if torch.accelerator.device_count() < world_size:
pytest.skip("GEMM-RS/AR requires two GPUs")
monkeypatch.setenv("NCCL_CUMEM_ENABLE", "1")
monkeypatch.setenv("NCCL_NVLS_ENABLE", "1")
try:
mp.spawn(
_worker,
args=(world_size, get_open_port(), backend),
nprocs=world_size,
)
finally:
cleanup_dist_env_and_memory()
@@ -1,670 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit test for the batched kpool decode-update kernel.
Validates ``kpool_decode_update_and_maybe_write_cache_batched`` against an
independent pure-torch reference that replicates the per-request, in-position
order semantics: stash each token into a paged tail ring; on pool completion
(``pos % pool_size == pool_size-1``) softmax(gate+ape)-weighted sum + Hadamard-128
+ per-vector fp8 absmax quant + write to the indexer K cache. Covers
no-completion, completion-at-end, completion-mid-batch, non-uniform padding,
plain decode, plus a randomized fuzz pass.
The kernel iterates each request's ``next_n`` tokens in position order inside
one program (grid = num_requests) to preserve the pool-completion
read-after-stash dependency; the reference mirrors that ordering.
``test_decode_writer_matches_prefill_writer`` is deliberately NOT
reference-based: it checks the decode writer against the *prefill* writer
(``kpool_compress_and_write_cache``), the invariant that actually matters in
production. A hand-written reference can drift to match a buggy kernel -- that
is exactly how the stash-gating bug (intra-pool tokens never entering the tail
ring, because the stash was gated on the pool-granular ``slot_mapping``) stayed
green here.
"""
import math
import pytest
import torch
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
if current_platform.is_rocm():
from vllm.models.glm5next.amd.ops.kpool_compress import (
kpool_compress_and_write_cache,
kpool_decode_update_and_maybe_write_cache_batched,
kpool_seed_tail_cache,
)
else:
from vllm.models.glm5next.nvidia.ops.kpool_compress import (
kpool_compress_and_write_cache,
kpool_decode_update_and_maybe_write_cache_batched,
kpool_seed_tail_cache,
)
HEAD_DIM = 128
POOL_SIZE = 16
PAGE_SIZE = 64
NUM_BLOCKS = 32
ROUND_SCALE = True
FP8_DTYPE = current_platform.fp8_dtype()
FP8_MAX = torch.finfo(FP8_DTYPE).max
def _make_caches():
kv = torch.zeros(
NUM_BLOCKS, PAGE_SIZE, HEAD_DIM + 4, dtype=torch.uint8, device="cuda"
)
tail = torch.zeros(
NUM_BLOCKS, 2, POOL_SIZE, HEAD_DIM, dtype=torch.bfloat16, device="cuda"
)
return kv, tail
def _tail_slot_for(blocks, pos):
"""tail_slot = block*POOL + pos%POOL; each request owns a distinct tail block."""
blk = torch.tensor(blocks, device=pos.device, dtype=torch.int32).unsqueeze(1)
return (blk * POOL_SIZE + pos % POOL_SIZE).to(torch.int32)
def _seed_prior(tail, blocks, n_prior, seed=42):
if n_prior <= 0:
return
g = torch.Generator(device=tail.device).manual_seed(seed)
prior_k = torch.randn(
len(blocks),
n_prior,
HEAD_DIM,
dtype=torch.bfloat16,
device=tail.device,
generator=g,
)
prior_s = torch.randn(
len(blocks),
n_prior,
HEAD_DIM,
dtype=torch.bfloat16,
device=tail.device,
generator=g,
)
for i, blk in enumerate(blocks):
tail[blk, 0, :n_prior, :] = prior_k[i]
tail[blk, 1, :n_prior, :] = prior_s[i]
def _hadamard128_torch(x: torch.Tensor) -> torch.Tensor:
"""Reference Hadamard-128 on the last dim (must be 128)."""
n = x.shape[-1]
assert n == 128
h = torch.tensor([[1.0, 1.0], [1.0, -1.0]], dtype=torch.float32, device=x.device)
while h.shape[0] < n:
h = torch.cat([torch.cat([h, h], dim=1), torch.cat([h, -h], dim=1)], dim=0)
h = h / math.sqrt(n)
return x @ h
def _torch_reference(
kv: torch.Tensor,
tail: torch.Tensor,
tail_slot: torch.Tensor,
key: torch.Tensor,
score: torch.Tensor,
ape: torch.Tensor,
slot_map: torch.Tensor,
pos: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Independent reference for the batched decode-update kernel.
For each request, iterate its next_n tokens in order. On each token:
- if pos%POOL == POOL-1 and pos_valid: compress the pool (slots
[pool_start..pool_start+POOL-1], current token via is_current) and write
fp8 K + fp32 scale to kv_cache at cache_loc.
- always stash the current token's K/score into tail[block, pos%POOL].
"""
kv = kv.clone()
tail = tail.clone()
B, next_n = pos.shape
# The indexer K cache is [num_blocks, PAGE_SIZE, HEAD_DIM+4] uint8 but the
# kernels interpret each page as [HEAD_DIM*PAGE_SIZE bytes of K (token-major)
# | 4*PAGE_SIZE bytes of fp32 scale (token-major)]. Operate on a flat byte
# view so the reference writes K and scale at the exact offsets the kernel
# uses (page_base + tok*HEAD_DIM for K; page_base + HEAD_DIM*PAGE_SIZE +
# tok*4 for scale).
page_bytes = PAGE_SIZE * (HEAD_DIM + 4)
k_region = HEAD_DIM * PAGE_SIZE
tail_slot_cpu = tail_slot.cpu().tolist()
slot_map_cpu = slot_map.cpu().tolist()
pos_cpu = pos.cpu().tolist()
key_cpu = key.float().cpu()
score_cpu = score.float().cpu()
ape_cpu = ape.cpu()
tail_cpu = tail.float().cpu()
kv_flat = kv.view(torch.uint8).reshape(-1).cpu()
for b in range(B):
for t in range(next_n):
cache_loc = slot_map_cpu[b][t]
p = pos_cpu[b][t]
pos_valid = cache_loc >= 0 and p >= 0
safe_pos = max(p, 0)
slot = safe_pos % POOL_SIZE
phys_slot = safe_pos % POOL_SIZE
# Per-token block derivation (a leading invalid sentinel must not
# poison the base for the rest of the request); clamped like the
# kernel so an invalid entry can't form a negative base.
block = max(tail_slot_cpu[b][t], 0) // POOL_SIZE
cur_key = key_cpu[b, t]
cur_score = score_cpu[b, t]
if pos_valid and slot == POOL_SIZE - 1:
pool_logical_start = safe_pos - slot
pool_scores = []
pool_ks = []
for ps in range(POOL_SIZE):
is_current = ps == slot
phys = (pool_logical_start + ps) % POOL_SIZE
if is_current:
s = cur_score
k = cur_key
else:
s = tail_cpu[block, 1, phys]
k = tail_cpu[block, 0, phys]
s = s + ape_cpu[ps]
pool_scores.append(s)
pool_ks.append(k)
pool_scores = torch.stack(pool_scores) # [POOL, D]
pool_ks = torch.stack(pool_ks) # [POOL, D]
max_score = pool_scores.max(dim=0).values
prob = torch.exp(pool_scores - max_score)
denom = prob.sum(dim=0)
acc = (pool_ks * prob).sum(dim=0)
x = (acc / denom).to(torch.bfloat16).to(torch.float32)
x = _hadamard128_torch(x).to(torch.bfloat16).to(torch.float32)
absmax = torch.clamp(x.abs().max(), min=1e-4)
if ROUND_SCALE:
scale = torch.exp2(torch.ceil(torch.log2(absmax / FP8_MAX)))
else:
scale = absmax / FP8_MAX
quantized = torch.clamp(x / scale, -FP8_MAX, FP8_MAX).to(FP8_DTYPE)
# write K and scale at the separated-layout offsets
loc = cache_loc
loc_page_index = loc // PAGE_SIZE
loc_tok = loc % PAGE_SIZE
page_base = loc_page_index * page_bytes
if current_platform.is_rocm():
dims = torch.arange(HEAD_DIM)
k_off = (
page_base
+ (loc_tok // 16) * 16 * HEAD_DIM
+ (dims // 16) * 16 * 16
+ (loc_tok % 16) * 16
+ dims % 16
)
else:
k_off = page_base + loc_tok * HEAD_DIM + torch.arange(HEAD_DIM)
s_off = page_base + k_region + loc_tok * 4
kv_flat[k_off] = quantized.view(torch.uint8)
scale_bytes = scale.detach().reshape(1).view(torch.uint8)
if mqa_logits_use_mx_scales():
# The power-of-two scale's exponent as 4 equal UE8M0 bytes.
ue8m0 = scale.reshape(1).view(torch.int32) >> 23
scale_bytes = ue8m0.to(torch.uint8).repeat(4)
kv_flat[s_off : s_off + 4] = scale_bytes
# stash -- gated on the TOKEN-granular tail slot, not on pos_valid.
# pos_valid keys off the pool-granular cache_loc, which is -1 for
# every token that is not the pool's last, so gating the stash on it
# would drop all intra-pool tokens.
if p >= 0 and tail_slot_cpu[b][t] >= 0:
tail_cpu[block, 0, phys_slot] = cur_key
tail_cpu[block, 1, phys_slot] = cur_score
kv_out = kv_flat.view(NUM_BLOCKS, PAGE_SIZE, HEAD_DIM + 4).to(device="cuda")
return kv_out, tail_cpu.to(torch.bfloat16).to(device="cuda")
def _assert_eq(r_ref, r_kern, k_byte_tol=0):
kv_ref, tail_ref = r_ref
kv_kern, tail_kern = r_kern
# Each page holds the fp8 K bytes followed by the fp32 scales.
k_region = PAGE_SIZE * HEAD_DIM
pages_ref = kv_ref.view(kv_ref.shape[0], -1)
pages_kern = kv_kern.view(kv_kern.shape[0], -1)
k_diff = (pages_ref[:, :k_region].int() - pages_kern[:, :k_region].int()).abs()
assert k_diff.max().item() <= k_byte_tol, (
f"kv_cache K differs: max diff {k_diff.max().item()}"
)
assert torch.equal(pages_ref[:, k_region:], pages_kern[:, k_region:]), (
"kv_cache scales differ"
)
assert torch.equal(tail_ref, tail_kern), (
"tail_kv_cache differs: max diff "
f"{(tail_ref.float() - tail_kern.float()).abs().max().item()}"
)
def _cache_pool_bytes(kv_cache: torch.Tensor, pool_slot: int) -> torch.Tensor:
"""Read a logical pool's K and scale bytes from the platform cache layout."""
page_size = kv_cache.shape[1]
head_dim = kv_cache.shape[2] - 4
page_idx, token_offset = divmod(pool_slot, page_size)
flat = kv_cache[page_idx].reshape(-1)
dims = torch.arange(head_dim, device=kv_cache.device)
if current_platform.is_rocm() and page_size > 1:
k_offsets = (
(token_offset // 16) * 16 * head_dim
+ (dims // 16) * 16 * 16
+ (token_offset % 16) * 16
+ dims % 16
)
else:
k_offsets = token_offset * head_dim + dims
scale_offset = page_size * head_dim + 4 * token_offset
return torch.cat((flat[k_offsets], flat[scale_offset : scale_offset + 4]))
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm required")
def test_amd_prefill_writer_uses_preshuffled_cache_layout():
from vllm.models.glm5next.amd.ops.kpool_compress import (
kpool_compress_and_write_cache as amd_kpool_compress,
)
torch.manual_seed(0)
token_offset = 17
kv = torch.zeros(1, PAGE_SIZE, HEAD_DIM + 4, dtype=torch.uint8, device="cuda")
key = torch.randn(1, POOL_SIZE, HEAD_DIM, dtype=torch.bfloat16, device="cuda")
score = torch.randn_like(key)
ape = torch.randn(POOL_SIZE, HEAD_DIM, dtype=torch.float32, device="cuda")
compressed_k, compressed_scale = amd_kpool_compress(
kv,
key,
score,
ape,
torch.tensor([token_offset], dtype=torch.int64, device="cuda"),
pool_size=POOL_SIZE,
head_dim=HEAD_DIM,
round_scale=ROUND_SCALE,
return_compressed=True,
)
dim = torch.arange(HEAD_DIM, device="cuda")
offsets = (
(token_offset // 16) * 16 * HEAD_DIM
+ (dim // 16) * 16 * 16
+ (token_offset % 16) * 16
+ dim % 16
)
flat = kv.view(torch.uint8).reshape(-1)
stored_k = flat[offsets].view(compressed_k.dtype)
scale_offset = PAGE_SIZE * HEAD_DIM + token_offset * 4
stored_scale = flat[scale_offset : scale_offset + 4].view(torch.float32)
assert torch.equal(stored_k, compressed_k[0])
assert torch.equal(stored_scale, compressed_scale)
def _run_kernel(kv, tail, tail_slot, key, score, ape, slot_map, pos):
kv = kv.clone()
tail = tail.clone()
kpool_decode_update_and_maybe_write_cache_batched(
kv,
tail,
tail_slot,
key,
score,
ape,
slot_map,
pos,
POOL_SIZE,
HEAD_DIM,
round_scale=ROUND_SCALE,
)
return kv, tail
@pytest.mark.parametrize("pool_size", [4, 16])
@pytest.mark.parametrize("ring_pools", [1, 2])
def test_decode_writer_matches_prefill_writer(pool_size, ring_pools):
ring = ring_pools * pool_size
n_pools, page, nblk = 8, 64, 4
n_tok = n_pools * pool_size
dev = "cuda"
torch.manual_seed(0)
k = torch.randn(n_tok, HEAD_DIM, dtype=torch.bfloat16, device=dev)
score = torch.randn(n_tok, HEAD_DIM, dtype=torch.bfloat16, device=dev)
ape = torch.randn(pool_size, HEAD_DIM, dtype=torch.float32, device=dev)
kv_prefill = torch.zeros(nblk, page, HEAD_DIM + 4, dtype=torch.uint8, device=dev)
kpool_compress_and_write_cache(
kv_prefill,
k.view(n_pools, pool_size, HEAD_DIM),
score.view(n_pools, pool_size, HEAD_DIM),
ape,
torch.arange(n_pools, dtype=torch.int64, device=dev),
pool_size=pool_size,
head_dim=HEAD_DIM,
round_scale=ROUND_SCALE,
)
# One request owning tail block 0, fed one token per decode step.
kv_decode = torch.zeros_like(kv_prefill)
tail = torch.zeros(nblk, 2, ring, HEAD_DIM, dtype=torch.bfloat16, device=dev)
for t in range(n_tok):
completes = t % pool_size == pool_size - 1
kpool_decode_update_and_maybe_write_cache_batched(
kv_decode,
tail,
# token-granular: every token has a valid tail slot
torch.tensor([[t % ring]], dtype=torch.int32, device=dev),
k[t].view(1, 1, HEAD_DIM),
score[t].view(1, 1, HEAD_DIM),
ape,
# pool-granular: only the pool's last token carries a cache slot
torch.tensor(
[[t // pool_size if completes else -1]], dtype=torch.int32, device=dev
),
torch.tensor([[t]], dtype=torch.int32, device=dev),
pool_size,
HEAD_DIM,
round_scale=ROUND_SCALE,
)
differing = [
p
for p in range(n_pools)
if not torch.equal(
_cache_pool_bytes(kv_prefill, p), _cache_pool_bytes(kv_decode, p)
)
]
assert not differing, (
f"decode-written pools differ from prefill-written pools: "
f"{len(differing)}/{n_pools} (pool_size={pool_size}, first={differing[:5]})"
)
@pytest.mark.parametrize("ring_pools", [1, 2])
def test_rejected_draft_redo_needs_ring_slots(ring_pools):
"""With a one-pool ring, the drafts behind a rejected pool-completing draft
overwrote the pool's earlier keys, so its redo compressed wrong keys."""
pool, spec, page, nblk = 4, 3, 64, 2
ring = ring_pools * pool
dev = "cuda"
torch.manual_seed(1)
n_tok = 3 * pool
k = torch.randn(n_tok, HEAD_DIM, dtype=torch.bfloat16, device=dev)
score = torch.randn(n_tok, HEAD_DIM, dtype=torch.bfloat16, device=dev)
ape = torch.randn(pool, HEAD_DIM, dtype=torch.float32, device=dev)
kv_ref = torch.zeros(nblk, page, HEAD_DIM + 4, dtype=torch.uint8, device=dev)
kpool_compress_and_write_cache(
kv_ref,
k.view(3, pool, HEAD_DIM),
score.view(3, pool, HEAD_DIM),
ape,
torch.arange(3, dtype=torch.int64, device=dev),
pool_size=pool,
head_dim=HEAD_DIM,
round_scale=ROUND_SCALE,
)
kv = torch.zeros_like(kv_ref)
tail = torch.zeros(nblk, 2, ring, HEAD_DIM, dtype=torch.bfloat16, device=dev)
def step(positions, keys, scores):
pos = torch.tensor([positions], dtype=torch.int32, device=dev)
slots = [(p // pool) if p % pool == pool - 1 else -1 for p in positions]
kpool_decode_update_and_maybe_write_cache_batched(
kv,
tail,
pos % ring,
keys.view(1, -1, HEAD_DIM),
scores.view(1, -1, HEAD_DIM),
ape,
torch.tensor([slots], dtype=torch.int32, device=dev),
pos,
pool,
HEAD_DIM,
round_scale=ROUND_SCALE,
)
for t in range(7):
step([t], k[t], score[t])
# Control: verified token 7 completes pool 1 before drafts 8..10 are stashed.
drafts = torch.randn(spec, HEAD_DIM, dtype=torch.bfloat16, device=dev)
draft_scores = torch.randn(spec, HEAD_DIM, dtype=torch.bfloat16, device=dev)
step(
[7, 8, 9, 10],
torch.cat([k[7:8], drafts]),
torch.cat([score[7:8], draft_scores]),
)
step([8, 9, 10, 11], k[8:12], score[8:12]) # all drafts rejected
for p in (1, 2):
assert torch.equal(_cache_pool_bytes(kv, p), _cache_pool_bytes(kv_ref, p)), p
# Draft 7 completes pool 1 and is rejected. With a one-pool ring, drafts
# 8 and 9 overwrite the slots of positions 4 and 5, which are read by the
# redo of 7.
kv.zero_()
tail.zero_()
for t in range(6):
step([t], k[t], score[t])
step(
[6, 7, 8, 9],
torch.cat([k[6:7], drafts]),
torch.cat([score[6:7], draft_scores]),
)
step([7, 8, 9, 10], k[7:11], score[7:11])
pool1_ok = torch.equal(_cache_pool_bytes(kv, 1), _cache_pool_bytes(kv_ref, 1))
if ring_pools == 1:
assert not pool1_ok, "expected a one-pool ring to corrupt pool 1"
else:
assert pool1_ok
def test_leading_invalid_tail_slot():
"""A request whose FIRST token carries an invalid (-1) tail slot while a
later token is a real pool completion.
The tail block must be derived per token, not from token 0: a leading
invalid sentinel would otherwise poison the base address for the whole
request (out-of-bounds tail reads on the completion).
"""
torch.manual_seed(0)
B, next_n, blocks = 2, 4, [3, 5]
# req 0: token 0 invalid (pos -1), tokens 1..3 valid, completion at pos 15
# req 1: all valid, no completion
pos = torch.tensor(
[[-1, 13, 14, 15], [4, 5, 6, 7]], dtype=torch.int32, device="cuda"
)
safe_pos = torch.where(pos >= 0, pos, 0)
tail_slot = _tail_slot_for(blocks, safe_pos)
# leading invalid entry carries the -1 sentinel, as the scatter path emits
tail_slot[0, 0] = -1
slot_map = torch.full((B, next_n), -1, dtype=torch.int32, device="cuda")
slot_map[0, 3] = 15 # req 0 completes its pool on the last verify token
key = torch.randn(B, next_n, HEAD_DIM, dtype=torch.bfloat16, device="cuda")
score = torch.randn(B, next_n, HEAD_DIM, dtype=torch.bfloat16, device="cuda")
ape = torch.randn(POOL_SIZE, HEAD_DIM, dtype=torch.float32, device="cuda")
kv, tail = _make_caches()
_seed_prior(tail, blocks, 13)
r_ref = _torch_reference(kv, tail, tail_slot, key, score, ape, slot_map, pos)
r_kern = _run_kernel(kv, tail, tail_slot, key, score, ape, slot_map, pos)
_assert_eq(r_ref, r_kern)
def test_prefill_seed_honors_padded_tail_block_stride():
"""The tail shares a padded indexer allocation in production.
``get_kv_cache_config_from_groups`` aliases each tail tensor onto its
indexer tensor with the indexer's block stride (38016 B for GLM-5.3-Flash
vs a dense 2048 B tail block), so a seed kernel that addresses blocks
densely writes into an unrelated indexer block and leaves the request's
tail block untouched. Runs on every platform; the NVIDIA kernel had this
bug while the AMD kernel did not.
"""
kpool = 4
num_blocks = 6
logical_block_elems = 2 * kpool * HEAD_DIM
padded_block_elems = logical_block_elems + 256
sentinel = -123.0
backing = torch.full(
(num_blocks * padded_block_elems,),
sentinel,
dtype=torch.bfloat16,
device="cuda",
)
tail = torch.as_strided(
backing,
size=(num_blocks, 2, kpool, HEAD_DIM),
stride=(padded_block_elems, kpool * HEAD_DIM, HEAD_DIM, 1),
)
block = 3
ring_slot = 2
key = torch.arange(HEAD_DIM, dtype=torch.bfloat16, device="cuda").unsqueeze(0)
score = (key + 256).to(torch.bfloat16)
tail_slot = torch.tensor(
[block * kpool + ring_slot], dtype=torch.int32, device="cuda"
)
kpool_seed_tail_cache(tail, key, score, tail_slot, kpool, HEAD_DIM)
torch.accelerator.synchronize()
assert torch.equal(tail[block, 0, ring_slot], key[0])
assert torch.equal(tail[block, 1, ring_slot], score[0])
compact_offset = (block * 2 * kpool + ring_slot) * HEAD_DIM
assert torch.all(backing[compact_offset : compact_offset + HEAD_DIM] == sentinel)
@pytest.mark.parametrize(
"case_id",
[
"no_completion",
"completion_at_end",
"completion_mid_batch",
"non_uniform_padding",
"plain_decode",
],
)
def test_batched_matches_reference(case_id):
torch.manual_seed(0)
if case_id == "no_completion":
B, next_n, blocks = 3, 4, [0, 1, 2]
pos = (
torch.arange(next_n, device="cuda", dtype=torch.int32)
.unsqueeze(0)
.expand(B, -1)
.contiguous()
)
slot_map = torch.full((B, next_n), -1, dtype=torch.int32, device="cuda")
n_prior = 0
elif case_id == "completion_at_end":
B, next_n, blocks = 2, 4, [0, 1]
pos = torch.tensor(
[[12, 13, 14, 15], [12, 13, 14, 15]], dtype=torch.int32, device="cuda"
)
slot_map = torch.full((B, next_n), -1, dtype=torch.int32, device="cuda")
slot_map[:, 3] = torch.tensor(
[15, PAGE_SIZE + 15], dtype=torch.int32, device="cuda"
)
n_prior = POOL_SIZE - next_n
elif case_id == "completion_mid_batch":
B, next_n, blocks = 3, 4, [0, 1, 2]
pos = torch.tensor([[13, 14, 15, 16]] * B, dtype=torch.int32, device="cuda")
slot_map = torch.full((B, next_n), -1, dtype=torch.int32, device="cuda")
slot_map[:, 2] = torch.tensor(
[15, PAGE_SIZE + 15, 2 * PAGE_SIZE + 15], dtype=torch.int32, device="cuda"
)
n_prior = 13
elif case_id == "non_uniform_padding":
B, next_n, blocks = 2, 4, [0, 1]
pos = torch.tensor(
[[12, 13, 14, 15], [12, 13, -1, -1]], dtype=torch.int32, device="cuda"
)
slot_map = torch.full((B, next_n), -1, dtype=torch.int32, device="cuda")
slot_map[0, 3] = 15
n_prior = POOL_SIZE - 4
else: # plain_decode
B, next_n, blocks = 4, 1, [0, 1, 2, 3]
pos = torch.tensor([[5], [6], [7], [8]], dtype=torch.int32, device="cuda")
slot_map = torch.full((B, next_n), -1, dtype=torch.int32, device="cuda")
n_prior = 0
if case_id == "non_uniform_padding":
safe_pos = torch.where(pos >= 0, pos, 0)
tail_slot = torch.where(pos >= 0, _tail_slot_for(blocks, safe_pos), 0)
else:
tail_slot = _tail_slot_for(blocks, pos)
key = torch.randn(B, next_n, HEAD_DIM, dtype=torch.bfloat16, device="cuda")
score = torch.randn(B, next_n, HEAD_DIM, dtype=torch.bfloat16, device="cuda")
ape = torch.randn(POOL_SIZE, HEAD_DIM, dtype=torch.float32, device="cuda")
kv, tail = _make_caches()
_seed_prior(tail, blocks, n_prior)
r_ref = _torch_reference(kv, tail, tail_slot, key, score, ape, slot_map, pos)
r_kern = _run_kernel(kv, tail, tail_slot, key, score, ape, slot_map, pos)
_assert_eq(r_ref, r_kern)
@pytest.mark.parametrize("seed", list(range(20)))
def test_batched_matches_reference_fuzz(seed):
"""Random B / next_n / start positions; covers 0, 1, and multi completion."""
g = torch.Generator(device="cuda").manual_seed(seed)
B = int(torch.randint(1, 6, (1,), generator=g, device="cuda").item())
next_n = int(torch.randint(1, 8, (1,), generator=g, device="cuda").item())
blocks = list(range(B))
starts = torch.randint(0, 33, (B,), generator=g, device="cuda", dtype=torch.int32)
pos = starts.unsqueeze(1) + torch.arange(
next_n, device="cuda", dtype=torch.int32
).unsqueeze(0)
tail_slot = _tail_slot_for(blocks, pos)
is_completion = pos % POOL_SIZE == POOL_SIZE - 1
blk = torch.tensor(blocks, device="cuda", dtype=torch.int32).unsqueeze(1)
pool_slot = blk * PAGE_SIZE + (POOL_SIZE - 1)
slot_map = torch.where(is_completion, pool_slot, torch.full_like(pos, -1))
key = torch.randn(
B, next_n, HEAD_DIM, dtype=torch.bfloat16, device="cuda", generator=g
)
score = torch.randn(
B, next_n, HEAD_DIM, dtype=torch.bfloat16, device="cuda", generator=g
)
ape = torch.randn(
POOL_SIZE, HEAD_DIM, dtype=torch.float32, device="cuda", generator=g
)
kv, tail = _make_caches()
prior_g = torch.Generator(device="cuda").manual_seed(seed + 1000)
for b in range(B):
n_prior = int(starts[b].item()) % POOL_SIZE
if n_prior > 0:
pk = torch.randn(
n_prior,
HEAD_DIM,
dtype=torch.bfloat16,
device="cuda",
generator=prior_g,
)
ps = torch.randn(
n_prior,
HEAD_DIM,
dtype=torch.bfloat16,
device="cuda",
generator=prior_g,
)
tail[blocks[b], 0, :n_prior, :] = pk
tail[blocks[b], 1, :n_prior, :] = ps
r_ref = _torch_reference(kv, tail, tail_slot, key, score, ape, slot_map, pos)
r_kern = _run_kernel(kv, tail, tail_slot, key, score, ape, slot_map, pos)
# Random inputs can land the pooled value on a bf16 rounding midpoint,
# where the kernel and the CPU reference may round apart by one fp8 step.
_assert_eq(r_ref, r_kern, k_byte_tol=1)
-609
View File
@@ -1,609 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for cuteDSL low-latency router GEMM (dot-product + split-K)."""
import pytest
import torch
import torch.nn.functional as F
pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
@pytest.fixture(autouse=True, scope="module")
def _require_sm90_and_cutedsl():
if torch.cuda.get_device_capability()[0] < 9:
pytest.skip("Requires SM90+ (Hopper/Blackwell)")
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import (
is_available,
)
if not is_available():
pytest.skip("cuteDSL (CUTLASS Python) not installed")
# ===== Helpers =====
def _ref(a, b):
return torch.mm(a.float(), b.float().T)
def _assert_close(out, ref, *, min_cos_sim=0.99, context=""):
assert out.device.type == "cuda", f"{context}: not on CUDA"
assert torch.isfinite(out).all(), f"{context}: NaN/Inf"
cos = F.cosine_similarity(
out.reshape(-1).float(), ref.reshape(-1).float(), dim=0
).item()
assert cos > min_cos_sim, (
f"{context}: cos_sim {cos:.6f} < {min_cos_sim} "
f"(abs_err={(out.float() - ref.float()).abs().max().item():.2e})"
)
def _can_precompile(a, b):
return (
a.dim() == 2
and b.dim() == 2
and a.dtype == torch.bfloat16
and b.dtype == torch.bfloat16
and a.device.type == "cuda"
and b.device.type == "cuda"
and a.device == b.device
and a.shape[1] == b.shape[1]
and a.is_contiguous()
and b.is_contiguous()
)
def _gemm(a, b):
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import (
ll_bf16_gemm,
ll_bf16_gemm_kernel,
)
if _can_precompile(a, b):
compile_key = ll_bf16_gemm_kernel.dispatch(
M=a.shape[0], K=a.shape[1], N=b.shape[0]
)
ll_bf16_gemm_kernel.compile(compile_key)
return ll_bf16_gemm(a, b)
def test_c1_pdl_kernel_is_selected_automatically(monkeypatch):
from vllm.model_executor.kernels.linear.cute_dsl import ll_bf16
calls = []
def default_kernel(hidden_states, router_weight, output_dtype):
calls.append(("default", hidden_states.shape[0]))
return hidden_states
def c1_pdl_kernel(hidden_states, router_weight, output_dtype):
calls.append(("c1_pdl", hidden_states.shape[0]))
return hidden_states
monkeypatch.setattr(ll_bf16, "ll_bf16_gemm_kernel", default_kernel)
monkeypatch.setattr(ll_bf16, "ll_bf16_gemm_c1_pdl_kernel", c1_pdl_kernel)
weight = torch.empty(4, 8, device="cuda", dtype=torch.bfloat16)
ll_bf16.ll_bf16_gemm(torch.empty(1, 8, device="cuda", dtype=torch.bfloat16), weight)
ll_bf16.ll_bf16_gemm(torch.empty(2, 8, device="cuda", dtype=torch.bfloat16), weight)
assert calls == [("c1_pdl", 1), ("default", 2)]
# ===== Shapes =====
SHAPES = [
(256, 7168, "DSV3"),
(256, 14400, "DSV4-Flash"),
(128, 5120, "DeepSeek-V2"),
(8, 4096, "Mixtral-8x7B"),
(64, 2880, "non-tile-aligned-K"),
(256, 2048, "split-K-boundary"),
]
SHAPES_SPLITK = [(n, k, d) for n, k, d in SHAPES if k >= 2048]
# =================================================================
# Dot-product kernel (M<=4 or K<2048)
# =================================================================
@pytest.mark.parametrize("M", [1, 2, 3, 4])
@pytest.mark.parametrize("N,K,desc", SHAPES, ids=[s[2] for s in SHAPES])
def test_dotprod(M, N, K, desc):
torch.manual_seed(42)
a = torch.randn(M, K, dtype=torch.bfloat16, device="cuda")
b = torch.randn(N, K, dtype=torch.bfloat16, device="cuda")
out = _gemm(a, b)
assert out.dtype == torch.float32
assert out.shape == (M, N)
_assert_close(out, _ref(a, b), context=f"dotprod {M}x{N}x{K}")
@pytest.mark.parametrize("M", [1, 4, 8, 16])
def test_dotprod_small_K(M):
"""K<2048 forces dot-product regardless of M."""
torch.manual_seed(42)
a = torch.randn(M, 1024, dtype=torch.bfloat16, device="cuda")
b = torch.randn(64, 1024, dtype=torch.bfloat16, device="cuda")
_assert_close(_gemm(a, b), _ref(a, b), context=f"small_K M={M}")
@pytest.mark.parametrize("K", [16, 32, 64, 128, 256, 512, 1024, 1536])
def test_dotprod_K_sweep(K):
torch.manual_seed(42)
a = torch.randn(4, K, dtype=torch.bfloat16, device="cuda")
b = torch.randn(32, K, dtype=torch.bfloat16, device="cuda")
_assert_close(_gemm(a, b), _ref(a, b), context=f"K={K}")
# =================================================================
# Split-K kernel (M>4 and K>=2048)
# =================================================================
@pytest.mark.parametrize("M", [5, 6, 8, 12, 16])
@pytest.mark.parametrize("N,K,desc", SHAPES_SPLITK, ids=[s[2] for s in SHAPES_SPLITK])
def test_splitk(M, N, K, desc):
torch.manual_seed(42)
a = torch.randn(M, K, dtype=torch.bfloat16, device="cuda")
b = torch.randn(N, K, dtype=torch.bfloat16, device="cuda")
out = _gemm(a, b)
assert out.dtype == torch.float32
assert out.shape == (M, N)
_assert_close(out, _ref(a, b), context=f"splitk {M}x{N}x{K}")
@pytest.mark.parametrize("K", [2048, 2304, 2880, 3072, 4096, 5120, 7168, 14400])
def test_splitk_K_sweep(K):
"""Includes non-tile-aligned K (2880) and uneven split (2304)."""
torch.manual_seed(42)
a = torch.randn(8, K, dtype=torch.bfloat16, device="cuda")
b = torch.randn(64, K, dtype=torch.bfloat16, device="cuda")
_assert_close(_gemm(a, b), _ref(a, b), context=f"splitk K={K}")
# =================================================================
# Dispatch boundary (M=4/5, K=2032/2048)
# =================================================================
@pytest.mark.parametrize("M", [4, 5])
@pytest.mark.parametrize("K", [2032, 2048])
def test_dispatch_boundary(M, K):
torch.manual_seed(42)
a = torch.randn(M, K, dtype=torch.bfloat16, device="cuda")
b = torch.randn(64, K, dtype=torch.bfloat16, device="cuda")
path = "splitk" if M > 4 and K >= 2048 else "dotprod"
_assert_close(_gemm(a, b), _ref(a, b), context=f"M={M} K={K} ({path})")
# =================================================================
# Arbitrary N
# =================================================================
@pytest.mark.parametrize("N", [1, 3, 7, 16, 17, 64, 128, 256, 384])
def test_arbitrary_N_dotprod(N):
torch.manual_seed(42)
a = torch.randn(4, 2048, dtype=torch.bfloat16, device="cuda")
b = torch.randn(N, 2048, dtype=torch.bfloat16, device="cuda")
out = _gemm(a, b)
assert out.shape == (4, N)
_assert_close(out, _ref(a, b), context=f"dotprod N={N}")
@pytest.mark.parametrize("N", [1, 8, 16, 17, 64, 128, 256])
def test_arbitrary_N_splitk(N):
torch.manual_seed(42)
a = torch.randn(8, 4096, dtype=torch.bfloat16, device="cuda")
b = torch.randn(N, 4096, dtype=torch.bfloat16, device="cuda")
out = _gemm(a, b)
assert out.shape == (8, N)
_assert_close(out, _ref(a, b), context=f"splitk N={N}")
# =================================================================
# Single token (M=1, decode path)
# =================================================================
@pytest.mark.parametrize(
"N,K",
[(256, 7168), (256, 14400), (8, 4096), (384, 7168), (264, 6144)],
ids=["DSV3", "DSV4-Flash", "Mixtral", "DSV4-Pro", "Inkling"],
)
def test_single_token(N, K):
torch.manual_seed(42)
a = torch.randn(1, K, dtype=torch.bfloat16, device="cuda")
b = torch.randn(N, K, dtype=torch.bfloat16, device="cuda")
out = _gemm(a, b)
assert out.shape == (1, N)
_assert_close(out, _ref(a, b), context=f"M=1 {N}x{K}")
def test_inkling_max_tokens():
torch.manual_seed(42)
a = torch.randn(64, 6144, dtype=torch.bfloat16, device="cuda")
b = torch.randn(264, 6144, dtype=torch.bfloat16, device="cuda")
out = _gemm(a, b)
assert out.shape == (64, 264)
_assert_close(out, _ref(a, b), context="Inkling M=64")
# =================================================================
# Numerical robustness
# =================================================================
@pytest.mark.parametrize("M,K", [(4, 2048), (8, 4096)], ids=["dotprod", "splitk"])
def test_large_values(M, K):
torch.manual_seed(42)
a = torch.randn(M, K, dtype=torch.bfloat16, device="cuda") * 100
b = torch.randn(64, K, dtype=torch.bfloat16, device="cuda") * 100
out = _gemm(a, b)
assert torch.isfinite(out).all()
_assert_close(out, _ref(a, b), context=f"large M={M}")
def test_near_zero():
torch.manual_seed(42)
a = torch.randn(4, 2048, dtype=torch.bfloat16, device="cuda") * 1e-4
b = torch.randn(64, 2048, dtype=torch.bfloat16, device="cuda") * 1e-4
out = _gemm(a, b)
assert torch.isfinite(out).all()
assert out.abs().max() < 1.0
def test_zeros():
a = torch.zeros(4, 2048, dtype=torch.bfloat16, device="cuda")
b = torch.randn(64, 2048, dtype=torch.bfloat16, device="cuda")
assert (_gemm(a, b) == 0).all()
def test_ones():
a = torch.ones(1, 2048, dtype=torch.bfloat16, device="cuda")
b = torch.randn(32, 2048, dtype=torch.bfloat16, device="cuda")
_assert_close(_gemm(a, b), _ref(a, b), context="ones")
# =================================================================
# Output dtype
# =================================================================
@pytest.mark.parametrize("M,K", [(4, 2048), (8, 4096)], ids=["dotprod", "splitk"])
def test_output_fp32(M, K):
torch.manual_seed(42)
a = torch.randn(M, K, dtype=torch.bfloat16, device="cuda")
b = torch.randn(64, K, dtype=torch.bfloat16, device="cuda")
assert _gemm(a, b).dtype == torch.float32
# =================================================================
# Determinism
# =================================================================
@pytest.mark.parametrize(
"M,K", [(1, 4096), (4, 4096), (5, 4096), (8, 4096), (16, 4096)]
)
def test_deterministic(M, K):
torch.manual_seed(42)
a = torch.randn(M, K, dtype=torch.bfloat16, device="cuda")
b = torch.randn(128, K, dtype=torch.bfloat16, device="cuda")
torch.testing.assert_close(_gemm(a, b), _gemm(a, b), atol=0, rtol=0)
# =================================================================
# Cross-kernel consistency
# =================================================================
@pytest.mark.parametrize("K", [2048, 4096, 7168])
def test_dotprod_vs_splitk(K):
"""First 4 rows from split-K (M=5) match dot-product (M=4)."""
torch.manual_seed(42)
a = torch.randn(5, K, dtype=torch.bfloat16, device="cuda")
b = torch.randn(64, K, dtype=torch.bfloat16, device="cuda")
_assert_close(
_gemm(a, b)[:4],
_gemm(a[:4], b),
min_cos_sim=0.999,
context=f"cross-kernel K={K}",
)
# =================================================================
# CUDA graph
# =================================================================
@pytest.mark.parametrize("M,K", [(4, 2048), (8, 4096)], ids=["dotprod", "splitk"])
def test_cudagraph(M, K):
torch.manual_seed(42)
a = torch.randn(M, K, dtype=torch.bfloat16, device="cuda")
b = torch.randn(64, K, dtype=torch.bfloat16, device="cuda")
_gemm(a, b)
torch.accelerator.synchronize()
g = torch.cuda.CUDAGraph()
with torch.cuda.graph(g):
out = _gemm(a, b)
for _ in range(5):
g.replay()
torch.accelerator.synchronize()
_assert_close(out, _ref(a, b), context=f"cudagraph M={M}")
def test_cudagraph_20x_replay():
torch.manual_seed(42)
a = torch.randn(4, 4096, dtype=torch.bfloat16, device="cuda")
b = torch.randn(256, 4096, dtype=torch.bfloat16, device="cuda")
_gemm(a, b)
torch.accelerator.synchronize()
g = torch.cuda.CUDAGraph()
with torch.cuda.graph(g):
out = _gemm(a, b)
results = []
for _ in range(20):
g.replay()
torch.accelerator.synchronize()
results.append(out.clone())
for i in range(1, len(results)):
torch.testing.assert_close(
results[0], results[i], atol=0, rtol=0, msg=f"Replay {i} differs"
)
def test_cudagraph_input_update():
torch.manual_seed(42)
a = torch.randn(4, 2048, dtype=torch.bfloat16, device="cuda")
b = torch.randn(64, 2048, dtype=torch.bfloat16, device="cuda")
_gemm(a, b)
torch.accelerator.synchronize()
g = torch.cuda.CUDAGraph()
with torch.cuda.graph(g):
out = _gemm(a, b)
a.copy_(torch.randn_like(a))
g.replay()
torch.accelerator.synchronize()
_assert_close(out, _ref(a, b), context="cudagraph input update")
# =================================================================
# GateLinear dispatch integration
# =================================================================
def _make_gate_linear(monkeypatch, *, params_dtype, out_dtype=torch.float32):
monkeypatch.setattr(
"vllm.model_executor.layers.linear.get_tensor_model_parallel_rank",
lambda: 0,
)
monkeypatch.setattr(
"vllm.model_executor.layers.linear.get_tensor_model_parallel_world_size",
lambda: 1,
)
monkeypatch.setattr(
"vllm.model_executor.parameter.get_tensor_model_parallel_rank",
lambda: 0,
)
monkeypatch.setattr(
"vllm.model_executor.parameter.get_tensor_model_parallel_world_size",
lambda: 1,
)
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.cute_dsl.ll_bf16.is_available",
lambda: True,
)
return GateLinear(
input_size=2048,
output_size=64,
bias=False,
out_dtype=out_dtype,
params_dtype=params_dtype,
).cuda()
def test_gate_linear_uses_ll_bf16_for_bf16_fast_path(monkeypatch):
gate = _make_gate_linear(monkeypatch, params_dtype=torch.bfloat16)
x = torch.randn(4, 2048, dtype=torch.bfloat16, device="cuda")
calls = []
def fake_ll_bf16_gemm(hidden_states, router_weight):
calls.append((hidden_states, router_weight))
return torch.full(
(hidden_states.shape[0], router_weight.shape[0]),
1.0,
dtype=torch.float32,
device=hidden_states.device,
)
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.cute_dsl.ll_bf16.ll_bf16_gemm",
fake_ll_bf16_gemm,
)
out, bias = gate(x)
assert bias is None
assert out.shape == (4, 64)
assert out.dtype == torch.float32
assert len(calls) == 1
assert calls[0][0] is x
assert calls[0][1] is gate.weight
def test_gate_linear_fp32_weight_falls_back(monkeypatch):
gate = _make_gate_linear(monkeypatch, params_dtype=torch.float32)
assert not gate.allow_ll_bf16_gemm
x = torch.randn(4, 2048, dtype=torch.bfloat16, device="cuda")
def fail_ll_bf16_gemm(hidden_states, router_weight):
raise AssertionError("ll_bf16_gemm should not run for fp32 weights")
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.cute_dsl.ll_bf16.ll_bf16_gemm",
fail_ll_bf16_gemm,
)
out, _ = gate(x)
assert out.shape == (4, 64)
assert out.dtype == torch.float32
def test_gate_linear_non_bf16_activation_falls_back(monkeypatch):
gate = _make_gate_linear(monkeypatch, params_dtype=torch.bfloat16)
x = torch.randn(4, 2048, dtype=torch.float16, device="cuda")
def fail_ll_bf16_gemm(hidden_states, router_weight):
raise AssertionError("ll_bf16_gemm should not run for non-bf16 activations")
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.cute_dsl.ll_bf16.ll_bf16_gemm",
fail_ll_bf16_gemm,
)
out, _ = gate(x)
assert out.shape == (4, 64)
assert out.dtype == torch.float32
def test_gate_linear_set_out_dtype_enables_ll_bf16(monkeypatch):
gate = _make_gate_linear(monkeypatch, params_dtype=torch.bfloat16, out_dtype=None)
assert not gate.allow_ll_bf16_gemm
gate.set_out_dtype(torch.float32)
assert gate.allow_ll_bf16_gemm
def test_gate_linear_non_fp32_out_dtype_disables_ll_bf16(monkeypatch):
gate = _make_gate_linear(monkeypatch, params_dtype=torch.bfloat16, out_dtype=None)
gate.set_out_dtype(torch.bfloat16)
assert not gate.allow_ll_bf16_gemm
def test_gate_linear_m_gt_16_falls_back(monkeypatch):
gate = _make_gate_linear(monkeypatch, params_dtype=torch.bfloat16)
x = torch.randn(17, 2048, dtype=torch.bfloat16, device="cuda")
def fail_ll_bf16_gemm(hidden_states, router_weight):
raise AssertionError("ll_bf16_gemm should not run for M > 16")
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.cute_dsl.ll_bf16.ll_bf16_gemm",
fail_ll_bf16_gemm,
)
out, _ = gate(x)
assert out.shape == (17, 64)
assert out.dtype == torch.float32
# =================================================================
# Negative tests — invalid inputs
# =================================================================
@pytest.mark.parametrize(
"M,K,N,dtype",
[
pytest.param(4, 2048, 64, torch.float32, id="fp32_input"),
pytest.param(4, 2048, 64, torch.float16, id="fp16_input"),
pytest.param(8, 4096, 64, torch.float32, id="fp32_splitk_path"),
],
)
def test_invalid_dtype(M, K, N, dtype):
a = torch.randn(M, K, device="cuda", dtype=dtype)
b = torch.randn(N, K, device="cuda", dtype=dtype)
with pytest.raises(ValueError, match="dtype=bfloat16"):
_gemm(a, b)
def test_invalid_device_cpu():
a = torch.randn(4, 2048, dtype=torch.bfloat16)
b = torch.randn(64, 2048, dtype=torch.bfloat16)
with pytest.raises(ValueError, match="device_type=cuda"):
_gemm(a, b)
def test_invalid_1d_input():
a = torch.randn(2048, dtype=torch.bfloat16, device="cuda")
b = torch.randn(64, 2048, dtype=torch.bfloat16, device="cuda")
with pytest.raises(ValueError, match="2D tensors"):
_gemm(a, b)
def test_mismatched_K():
a = torch.randn(4, 2048, dtype=torch.bfloat16, device="cuda")
b = torch.randn(64, 1024, dtype=torch.bfloat16, device="cuda")
with pytest.raises(ValueError, match="matching K dimensions"):
_gemm(a, b)
def test_invalid_K_divisibility():
a = torch.randn(4, 2049, dtype=torch.bfloat16, device="cuda")
b = torch.randn(64, 2049, dtype=torch.bfloat16, device="cuda")
with pytest.raises(ValueError, match="K to be divisible by 8"):
_gemm(a, b)
def test_non_contiguous_input():
a = torch.randn(2048, 4, dtype=torch.bfloat16, device="cuda").T
b = torch.randn(64, 2048, dtype=torch.bfloat16, device="cuda")
assert not a.is_contiguous()
with pytest.raises(ValueError, match="contiguous row-major"):
_gemm(a, b)
def test_invalid_output_dtype():
a = torch.randn(4, 2048, dtype=torch.bfloat16, device="cuda")
b = torch.randn(64, 2048, dtype=torch.bfloat16, device="cuda")
with pytest.raises(ValueError, match="output_dtype=torch.float32"):
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import ll_bf16_gemm
ll_bf16_gemm(a, b, output_dtype=torch.float16)
def test_cache_miss_compiles_dotprod():
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import LLBf16Gemm
torch.manual_seed(42)
a = torch.randn(3, 64, dtype=torch.bfloat16, device="cuda")
b = torch.randn(17, 64, dtype=torch.bfloat16, device="cuda")
kernel = LLBf16Gemm()
out = kernel(a, b)
assert out.shape == (3, 17)
_assert_close(out, _ref(a, b), context="cache miss dotprod")
@pytest.mark.parametrize("M", [1, 2, 8, 16])
@pytest.mark.parametrize("N", [1792, 3584])
@torch.inference_mode()
def test_bf16_output_and_fp32_cache_coexist(M, N):
"""BF16 epilogues retain FP32 accumulation and can replay with new inputs."""
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import ll_bf16_gemm
torch.manual_seed(42)
x = torch.randn(M, 5120, device="cuda", dtype=torch.bfloat16)
w = torch.randn(N, 5120, device="cuda", dtype=torch.bfloat16) * 0.01
for dtype in (torch.float32, torch.bfloat16):
actual = ll_bf16_gemm(x, w, output_dtype=dtype)
assert actual.dtype == dtype
reference = (x.double() @ w.double().T).to(dtype)
torch.testing.assert_close(actual, reference, rtol=0.008, atol=1e-5)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
actual = ll_bf16_gemm(x, w, output_dtype=torch.bfloat16)
x.normal_()
graph.replay()
reference = (x.double() @ w.double().T).bfloat16()
torch.testing.assert_close(actual, reference, rtol=0.008, atol=1e-5)
if __name__ == "__main__":
pytest.main([__file__, "-v"])
-544
View File
@@ -1,544 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for cuteDSL low-latency GEMM with fp32 weights."""
import pytest
import torch
import torch.nn.functional as F
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
not torch.accelerator.is_available(), reason="CUDA required"
)
@pytest.fixture(autouse=True, scope="module")
def _require_sm90_and_cutedsl():
if not current_platform.has_device_capability(90):
pytest.skip("Requires SM90+ (Hopper/Blackwell)")
from vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w import (
is_available,
)
if not is_available():
pytest.skip("cuteDSL (CUTLASS Python) not installed")
def _ref(a, b):
return torch.mm(a.float(), b.float().T)
def _assert_close(out, ref, *, min_cos_sim=0.99, context=""):
assert out.device.type == "cuda", f"{context}: not on CUDA"
assert torch.isfinite(out).all(), f"{context}: NaN/Inf"
cos = F.cosine_similarity(
out.reshape(-1).float(), ref.reshape(-1).float(), dim=0
).item()
assert cos > min_cos_sim, (
f"{context}: cos_sim {cos:.6f} < {min_cos_sim} "
f"(abs_err={(out.float() - ref.float()).abs().max().item():.2e})"
)
def _gemm(a, b, output_dtype=torch.float32):
from vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w import (
LL_FP32W_GEMM_KERNEL,
)
return LL_FP32W_GEMM_KERNEL(a, b, output_dtype)
SHAPES = [
(256, 3072, "MiniMax-M2"),
(256, 7168, "DSV3-sized"),
(128, 5120, "DeepSeek-V2"),
(64, 2880, "non-aligned-K"),
(8, 4096, "Mixtral"),
]
A_DTYPES = [
pytest.param(torch.bfloat16, id="bf16"),
pytest.param(torch.float16, id="fp16"),
pytest.param(torch.float32, id="fp32"),
]
# =================================================================
# Correctness: all dtype combos x shapes x M values
# =================================================================
@pytest.mark.parametrize("a_dtype", A_DTYPES)
@pytest.mark.parametrize("M", [1, 2, 4, 8, 16])
@pytest.mark.parametrize("N,K,desc", SHAPES, ids=[s[2] for s in SHAPES])
def test_correctness(M, N, K, desc, a_dtype):
torch.manual_seed(42)
a = torch.randn(M, K, device="cuda").to(a_dtype)
b = torch.randn(N, K, dtype=torch.float32, device="cuda")
out = _gemm(a, b)
assert out.dtype == torch.float32
assert out.shape == (M, N)
_assert_close(out, _ref(a, b), context=f"{a_dtype} M={M} {desc}")
# =================================================================
# Small K (k_main=0, only tail/scalar paths)
# =================================================================
@pytest.mark.parametrize("K", [16, 32, 64, 128, 256, 512, 1024, 1536])
def test_small_K(K):
torch.manual_seed(42)
a = torch.randn(4, K, dtype=torch.bfloat16, device="cuda")
b = torch.randn(64, K, dtype=torch.float32, device="cuda")
_assert_close(_gemm(a, b), _ref(a, b), context=f"small_K={K}")
# =================================================================
# K sweep (covers main loop + various tail combinations)
# =================================================================
@pytest.mark.parametrize("K", [2048, 2304, 2880, 3072, 4096, 5120, 7168])
def test_K_sweep(K):
torch.manual_seed(42)
a = torch.randn(8, K, dtype=torch.bfloat16, device="cuda")
b = torch.randn(64, K, dtype=torch.float32, device="cuda")
_assert_close(_gemm(a, b), _ref(a, b), context=f"K={K}")
# =================================================================
# MiniMax-M2 specific shapes
# =================================================================
@pytest.mark.parametrize("M", [1, 4, 8, 16, 32])
def test_minimax_m2(M):
torch.manual_seed(42)
a = torch.randn(M, 3072, dtype=torch.bfloat16, device="cuda")
b = torch.randn(256, 3072, dtype=torch.float32, device="cuda")
_assert_close(_gemm(a, b), _ref(a, b), context=f"MiniMax-M2 M={M}")
# =================================================================
# Single token (M=1, decode path)
# =================================================================
@pytest.mark.parametrize(
"N,K",
[(256, 3072), (256, 7168), (8, 4096)],
ids=["MiniMax-M2", "DSV3-sized", "Mixtral"],
)
def test_single_token(N, K):
torch.manual_seed(42)
a = torch.randn(1, K, dtype=torch.bfloat16, device="cuda")
b = torch.randn(N, K, dtype=torch.float32, device="cuda")
out = _gemm(a, b)
assert out.shape == (1, N)
_assert_close(out, _ref(a, b), context=f"M=1 {N}x{K}")
# =================================================================
# Arbitrary N
# =================================================================
@pytest.mark.parametrize("N", [1, 3, 16, 64, 256])
def test_arbitrary_N(N):
torch.manual_seed(42)
a = torch.randn(4, 3072, dtype=torch.bfloat16, device="cuda")
b = torch.randn(N, 3072, dtype=torch.float32, device="cuda")
out = _gemm(a, b)
assert out.shape == (4, N)
_assert_close(out, _ref(a, b), context=f"N={N}")
# =================================================================
# Numerical robustness
# =================================================================
def test_large_values():
torch.manual_seed(42)
a = torch.randn(4, 5120, dtype=torch.bfloat16, device="cuda") * 100
b = torch.randn(128, 5120, dtype=torch.float32, device="cuda") * 100
out = _gemm(a, b)
assert torch.isfinite(out).all()
_assert_close(out, _ref(a, b), context="large_values")
def test_near_zero():
torch.manual_seed(42)
a = torch.randn(4, 4096, dtype=torch.bfloat16, device="cuda") * 1e-4
b = torch.randn(64, 4096, dtype=torch.float32, device="cuda") * 1e-4
out = _gemm(a, b)
assert torch.isfinite(out).all()
assert out.abs().max() < 1.0
def test_zeros():
a = torch.zeros(4, 2048, dtype=torch.bfloat16, device="cuda")
b = torch.randn(64, 2048, dtype=torch.float32, device="cuda")
assert (_gemm(a, b) == 0).all()
def test_ones():
a = torch.ones(1, 4096, dtype=torch.bfloat16, device="cuda")
b = torch.randn(32, 4096, dtype=torch.float32, device="cuda")
_assert_close(_gemm(a, b), _ref(a, b), context="ones")
# =================================================================
# Output dtype
# =================================================================
def test_output_fp32():
torch.manual_seed(42)
a = torch.randn(4, 5120, dtype=torch.bfloat16, device="cuda")
b = torch.randn(128, 5120, dtype=torch.float32, device="cuda")
assert _gemm(a, b).dtype == torch.float32
# =================================================================
# Determinism
# =================================================================
@pytest.mark.parametrize("M", [1, 4, 8, 16])
@pytest.mark.parametrize("K", [1024, 3072, 7168])
def test_deterministic(M, K):
torch.manual_seed(42)
a = torch.randn(M, K, dtype=torch.bfloat16, device="cuda")
b = torch.randn(64, K, dtype=torch.float32, device="cuda")
torch.testing.assert_close(_gemm(a, b), _gemm(a, b), atol=0, rtol=0)
# =================================================================
# Cross-dtype consistency
# =================================================================
def test_bf16_vs_fp32_activations():
"""bf16 and fp32 activations on same data should roughly agree."""
torch.manual_seed(42)
K, N = 4096, 128
a_bf16 = torch.randn(4, K, dtype=torch.bfloat16, device="cuda")
b = torch.randn(N, K, dtype=torch.float32, device="cuda")
out_bf16 = _gemm(a_bf16, b)
a_fp32 = a_bf16.float()
out_fp32 = _gemm(a_fp32, b)
_assert_close(
out_bf16, out_fp32, min_cos_sim=0.999, context="bf16_vs_fp32_activations"
)
# =================================================================
# CUDA graph
# =================================================================
@pytest.mark.parametrize(
"M,K,N", [(4, 3072, 256), (16, 5120, 128)], ids=["MiniMax", "DeepSeek"]
)
def test_cudagraph(M, K, N):
torch.manual_seed(42)
a = torch.randn(M, K, dtype=torch.bfloat16, device="cuda")
b = torch.randn(N, K, dtype=torch.float32, device="cuda")
_gemm(a, b)
torch.accelerator.synchronize()
g = torch.cuda.CUDAGraph()
with torch.cuda.graph(g):
out = _gemm(a, b)
for _ in range(5):
g.replay()
torch.accelerator.synchronize()
_assert_close(out, _ref(a, b), context=f"cudagraph M={M} K={K}")
def test_cudagraph_20x_replay():
torch.manual_seed(42)
a = torch.randn(4, 4096, dtype=torch.bfloat16, device="cuda")
b = torch.randn(64, 4096, dtype=torch.float32, device="cuda")
_gemm(a, b)
torch.accelerator.synchronize()
g = torch.cuda.CUDAGraph()
with torch.cuda.graph(g):
out = _gemm(a, b)
results = []
for _ in range(20):
g.replay()
torch.accelerator.synchronize()
results.append(out.clone())
for i in range(1, len(results)):
torch.testing.assert_close(
results[0], results[i], atol=0, rtol=0, msg=f"Replay {i} differs"
)
def test_cudagraph_input_update():
torch.manual_seed(42)
a = torch.randn(4, 5120, dtype=torch.bfloat16, device="cuda")
b = torch.randn(128, 5120, dtype=torch.float32, device="cuda")
_gemm(a, b)
torch.accelerator.synchronize()
g = torch.cuda.CUDAGraph()
with torch.cuda.graph(g):
out = _gemm(a, b)
a.copy_(torch.randn_like(a))
g.replay()
torch.accelerator.synchronize()
_assert_close(out, _ref(a, b), context="cudagraph input update")
# =================================================================
# Negative tests
# =================================================================
def test_invalid_weight_dtype():
a = torch.randn(4, 4096, dtype=torch.bfloat16, device="cuda")
b = torch.randn(64, 4096, dtype=torch.bfloat16, device="cuda")
with pytest.raises(ValueError):
_gemm(a, b)
def test_invalid_device_cpu():
a = torch.randn(4, 4096, dtype=torch.bfloat16)
b = torch.randn(64, 4096, dtype=torch.float32)
with pytest.raises(ValueError):
_gemm(a, b)
def test_invalid_1d_input():
a = torch.randn(4096, dtype=torch.bfloat16, device="cuda")
b = torch.randn(64, 4096, dtype=torch.float32, device="cuda")
with pytest.raises(ValueError):
_gemm(a, b)
def test_invalid_activation_dtype():
a = torch.ones(4, 4096, dtype=torch.int32, device="cuda")
b = torch.randn(64, 4096, dtype=torch.float32, device="cuda")
with pytest.raises(ValueError):
_gemm(a, b)
def test_invalid_output_dtype():
a = torch.randn(4, 4096, dtype=torch.bfloat16, device="cuda")
b = torch.randn(64, 4096, dtype=torch.float32, device="cuda")
with pytest.raises(ValueError):
_gemm(a, b, torch.bfloat16)
def test_invalid_k_mismatch():
a = torch.randn(4, 4096, dtype=torch.bfloat16, device="cuda")
b = torch.randn(64, 2048, dtype=torch.float32, device="cuda")
with pytest.raises(ValueError):
_gemm(a, b)
def test_invalid_non_contiguous():
base = torch.randn(4, 8192, dtype=torch.bfloat16, device="cuda")
a = base[:, ::2]
b = torch.randn(64, 4096, dtype=torch.float32, device="cuda")
with pytest.raises(ValueError):
_gemm(a, b)
# =================================================================
# GateLinear dispatch integration
# =================================================================
def _make_gate_linear_fp32w(
monkeypatch,
*,
params_dtype,
out_dtype=torch.float32,
input_size=2048,
output_size=64,
):
monkeypatch.setattr(
"vllm.model_executor.layers.linear.get_tensor_model_parallel_rank",
lambda: 0,
)
monkeypatch.setattr(
"vllm.model_executor.layers.linear.get_tensor_model_parallel_world_size",
lambda: 1,
)
monkeypatch.setattr(
"vllm.model_executor.parameter.get_tensor_model_parallel_rank",
lambda: 0,
)
monkeypatch.setattr(
"vllm.model_executor.parameter.get_tensor_model_parallel_world_size",
lambda: 1,
)
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.cute_dsl.ll_bf16.is_available",
lambda: False,
)
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w.is_available",
lambda: True,
)
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
return GateLinear(
input_size=input_size,
output_size=output_size,
bias=False,
out_dtype=out_dtype,
params_dtype=params_dtype,
).cuda()
@pytest.mark.parametrize("x_dtype", [torch.bfloat16, torch.float16, torch.float32])
def test_gate_linear_uses_ll_fp32w_for_fp32_weight_fast_path(monkeypatch, x_dtype):
gate = _make_gate_linear_fp32w(monkeypatch, params_dtype=torch.float32)
x = torch.randn(4, 2048, dtype=x_dtype, device="cuda")
calls = []
def fake_ll_fp32w_gemm(hidden_states, router_weight):
calls.append((hidden_states, router_weight))
return torch.full(
(hidden_states.shape[0], router_weight.shape[0]),
1.0,
dtype=torch.float32,
device=hidden_states.device,
)
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w.LL_FP32W_GEMM_KERNEL",
fake_ll_fp32w_gemm,
)
out, bias = gate(x)
assert bias is None
assert out.shape == (4, 64)
assert out.dtype == torch.float32
assert len(calls) == 1
assert calls[0][0] is x
assert calls[0][1] is gate.weight
PREFERRED_FP32W_GATE_CASES = [
*((6144, 128, M) for M in range(1, 33)),
*((6144, 256, M) for M in (1, 8, 12, 14, 16, 20, 22, 25, 27, 32)),
]
@pytest.mark.parametrize(("input_size", "output_size", "M"), PREFERRED_FP32W_GATE_CASES)
def test_gate_linear_uses_ll_fp32w_when_cpp_tier_disabled(
monkeypatch, input_size, output_size, M
):
gate = _make_gate_linear_fp32w(
monkeypatch,
params_dtype=torch.float32,
input_size=input_size,
output_size=output_size,
)
gate.allow_fp32_router_gemm = False
x = torch.randn(M, input_size, dtype=torch.bfloat16, device="cuda")
calls = []
def fake_ll_fp32w_gemm(hidden_states, router_weight):
calls.append((hidden_states, router_weight))
return torch.full(
(hidden_states.shape[0], router_weight.shape[0]),
2.0,
dtype=torch.float32,
device=hidden_states.device,
)
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w.LL_FP32W_GEMM_KERNEL",
fake_ll_fp32w_gemm,
)
out, _ = gate(x)
assert len(calls) == 1
assert calls[0][0] is x
assert calls[0][1] is gate.weight
assert torch.all(out == 2.0)
@pytest.mark.parametrize("M", [8, 16, 32])
def test_gate_linear_uses_cpp_fp32_router_for_supported_cpp_shape(monkeypatch, M):
gate = _make_gate_linear_fp32w(
monkeypatch,
params_dtype=torch.float32,
input_size=3072,
output_size=256,
)
assert gate.allow_fp32_router_gemm
x = torch.randn(M, 3072, dtype=torch.bfloat16, device="cuda")
def fail_ll_fp32w_gemm(hidden_states, router_weight):
raise AssertionError(
"ll_fp32w_gemm should not bypass fp32_router_gemm_dispatch"
)
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w.LL_FP32W_GEMM_KERNEL",
fail_ll_fp32w_gemm,
)
out, _ = gate(x)
assert out.shape == (M, 256)
assert out.dtype == torch.float32
def test_gate_linear_bf16_weight_skips_ll_fp32w(monkeypatch):
gate = _make_gate_linear_fp32w(monkeypatch, params_dtype=torch.bfloat16)
assert not gate.allow_ll_fp32w_gemm
x = torch.randn(4, 2048, dtype=torch.bfloat16, device="cuda")
def fail_ll_fp32w_gemm(hidden_states, router_weight):
raise AssertionError("ll_fp32w_gemm should not run for bf16 weights")
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w.LL_FP32W_GEMM_KERNEL",
fail_ll_fp32w_gemm,
)
out, _ = gate(x)
assert out.shape == (4, 64)
assert out.dtype == torch.float32
def test_gate_linear_set_out_dtype_enables_ll_fp32w(monkeypatch):
gate = _make_gate_linear_fp32w(
monkeypatch, params_dtype=torch.float32, out_dtype=None
)
assert not gate.allow_ll_fp32w_gemm
gate.set_out_dtype(torch.float32)
assert gate.allow_ll_fp32w_gemm
def test_gate_linear_non_fp32_out_dtype_disables_ll_fp32w(monkeypatch):
gate = _make_gate_linear_fp32w(
monkeypatch, params_dtype=torch.float32, out_dtype=None
)
gate.set_out_dtype(torch.bfloat16)
assert not gate.allow_ll_fp32w_gemm
def test_gate_linear_m_gt_32_skips_ll_fp32w(monkeypatch):
gate = _make_gate_linear_fp32w(monkeypatch, params_dtype=torch.float32)
x = torch.randn(33, 2048, dtype=torch.bfloat16, device="cuda")
def fail_ll_fp32w_gemm(hidden_states, router_weight):
raise AssertionError("ll_fp32w_gemm should not run for M > 32")
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w.LL_FP32W_GEMM_KERNEL",
fail_ll_fp32w_gemm,
)
out, _ = gate(x)
assert out.shape == (33, 64)
assert out.dtype == torch.float32
if __name__ == "__main__":
pytest.main([__file__, "-v"])
-688
View File
@@ -1,688 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Reference-vs-optimized unit tests for the MiniMax-M3 AMD/ROCm fused kernels.
Each optimized kernel added for the ROCm port has a slow PyTorch reference; the
tests assert the two agree within tolerance:
* Gemma RMSNorm (plain + fused-add-residual) -> fp32 PyTorch normalize
* SwiGLU-OAI (split layout) -> fp32 PyTorch elementwise
* Fused MXFP8 activation quant (Triton) -> _mxfp8_e4m3_quantize_torch
* Native MXFP8 linear (dot_scaled) -> dequant-to-bf16 @ matmul
* Native MXFP8 MoE (dot_scaled grouped GEMM) -> dequant-to-bf16 MoE math
* Strided K/V sparse-PA cache insert -> contiguous-copy insert
The native MXFP8 GEMMs also guard the ``dot_scaled`` rhs-scale orientation: the
scale is loaded ``[N, K//32]`` and passed WITHOUT transpose; a stray ``.T``
makes the shape ``[K//32, N]`` and Triton raises before producing output, so any
regression there fails these tests loudly.
Hardware scope: the whole module is ROCm-only (these are the AMD path; NVIDIA
uses the FlashInfer kernels). The norm/activation/quant kernels run on any ROCm
arch; the native MXFP8 ``dot_scaled`` linear/MoE tests are additionally gated to
CDNA4 gfx95x (``@requires_gfx950``) since gfx942 uses the BF16 emulation path.
Run: pytest tests/kernels/test_minimax_m3_amd_ops.py -v
"""
import pytest
import torch
from vllm.platforms import current_platform
if not current_platform.is_rocm():
pytest.skip("MiniMax-M3 AMD fused ops require ROCm.", allow_module_level=True)
if not torch.cuda.is_available():
pytest.skip("Requires a GPU.", allow_module_level=True)
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( # noqa: E402
_mxfp8_e4m3_quantize_torch,
_mxfp8_e4m3_quantize_triton,
dequant_mxfp8_to_bf16,
)
from vllm.models.minimax_m3.amd.model import _kv_insert_operand # noqa: E402
from vllm.models.minimax_m3.amd.ops import ( # noqa: E402
gemma_fused_add_rmsnorm,
gemma_rmsnorm,
swiglu_oai_quantize_mxfp8,
swiglu_oai_split,
)
from vllm.models.minimax_m3.amd.ops.gemma_rmsnorm import _num_warps # noqa: E402
DEVICE = "cuda"
EPS = 1e-6
def _gcn_arch() -> str:
try:
return torch.cuda.get_device_properties(0).gcnArchName
except Exception: # pragma: no cover - no device / non-AMD
return ""
# The pure-Triton norm/activation/quant kernels run on any ROCm arch (CDNA3
# gfx942 and CDNA4 gfx950). The native MXFP8 ``dot_scaled`` GEMMs (linear + MoE)
# use CDNA4 hardware microscaling and are gated to gfx95x in the source
# (``RocmDotScaledMxfp8LinearKernel.is_supported``; the MoE oracle routes gfx942
# to the BF16 emulation path instead) — so those tests are gfx950-only.
requires_gfx950 = pytest.mark.skipif(
"gfx95" not in _gcn_arch(),
reason="native MXFP8 dot_scaled is a CDNA4 (gfx95x) feature; "
"gfx942 uses the BF16 emulation path instead.",
)
def _relerr(a: torch.Tensor, b: torch.Tensor) -> float:
a = a.float()
b = b.float()
return ((a - b).norm() / (b.norm() + 1e-8)).item()
# --------------------------------------------------------------------------- #
# Gemma RMSNorm
# --------------------------------------------------------------------------- #
def _ref_gemma_rmsnorm(x, w, eps, residual=None):
orig_dtype = x.dtype
xf = x.float()
res_out = None
if residual is not None:
xf = xf + residual.float()
res_out = xf.to(orig_dtype)
xf = xf * torch.rsqrt(xf.pow(2).mean(dim=-1, keepdim=True) + eps)
xf = xf * (1.0 + w.float())
out = xf.to(orig_dtype)
return out if residual is None else (out, res_out)
@pytest.mark.parametrize("shape", [(1, 4096), (37, 6144), (128, 2048)])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("seed", [0, 1234])
@torch.inference_mode()
def test_gemma_rmsnorm(shape, dtype, seed):
torch.manual_seed(seed)
x = torch.randn(*shape, device=DEVICE, dtype=dtype)
w = torch.randn(shape[-1], device=DEVICE, dtype=dtype) * 0.1
got = gemma_rmsnorm(x, w, EPS)
ref = _ref_gemma_rmsnorm(x, w, EPS)
assert got.shape == x.shape
assert _relerr(got, ref) < 5e-3
@pytest.mark.parametrize("shape", [(1, 6144), (64, 4096)])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@torch.inference_mode()
def test_gemma_fused_add_rmsnorm(shape, dtype):
torch.manual_seed(0)
x = torch.randn(*shape, device=DEVICE, dtype=dtype)
res = torch.randn(*shape, device=DEVICE, dtype=dtype)
w = torch.randn(shape[-1], device=DEVICE, dtype=dtype) * 0.1
got_out, got_res = gemma_fused_add_rmsnorm(x, res, w, EPS)
ref_out, ref_res = _ref_gemma_rmsnorm(x, w, EPS, residual=res)
assert _relerr(got_out, ref_out) < 5e-3
# residual_out is the pre-norm sum (x + res): bit-for-bit identical cast.
assert torch.equal(got_res, ref_res)
@torch.inference_mode()
def test_gemma_rmsnorm_per_head_strided():
"""q_norm/k_norm normalize a non-contiguous ``qkv.split`` slice over head_dim."""
torch.manual_seed(0)
T, H, D, kv = 7, 48, 128, 8
total = (H + 2 * kv) * D
qkv = torch.randn(T, total, device=DEVICE, dtype=torch.bfloat16)
q = qkv[..., : H * D] # non-contiguous view (row stride == total)
q_by_head = q.view(T, H, D)
assert not q_by_head.is_contiguous()
w = torch.randn(D, device=DEVICE, dtype=torch.bfloat16) * 0.1
got = gemma_rmsnorm(q_by_head, w, EPS)
ref = _ref_gemma_rmsnorm(q_by_head, w, EPS)
assert got.shape == q_by_head.shape
assert _relerr(got, ref) < 5e-3
def test_num_warps_monotonic():
assert _num_warps(128) <= _num_warps(2048) <= _num_warps(8192)
# --------------------------------------------------------------------------- #
# SwiGLU-OAI (split layout)
# --------------------------------------------------------------------------- #
def _ref_swiglu(gate_up, alpha, beta, limit):
d = gate_up.shape[-1] // 2
gate = gate_up[..., :d].float()
up = gate_up[..., d:].float()
if limit is not None:
gate = gate.clamp(max=limit)
up = up.clamp(min=-limit, max=limit)
return (gate * torch.sigmoid(alpha * gate) * (up + beta)).to(gate_up.dtype)
@pytest.mark.parametrize("m,inter", [(1, 768), (64, 1536), (128, 1024)])
@pytest.mark.parametrize("limit", [7.0, None])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@torch.inference_mode()
def test_swiglu_oai_split(m, inter, limit, dtype):
torch.manual_seed(0)
gate_up = torch.randn(m, 2 * inter, device=DEVICE, dtype=dtype)
got = swiglu_oai_split(gate_up, alpha=1.702, beta=1.0, limit=limit)
ref = _ref_swiglu(gate_up, 1.702, 1.0, limit)
assert got.shape == (m, inter)
assert _relerr(got, ref) < 5e-3
@torch.inference_mode()
def test_swiglu_oai_quantize_mxfp8_uses_e4m3_range_for_scale():
# Keep a small value in two MX blocks next to their maxima. The scale must
# use E4M3's finite range (448), otherwise that value underflows to zero.
# Keep the third block empty to cover the reference's finite tiny clamp.
m, inter = 3, 96
gate = torch.full((m, inter), 2.0, device=DEVICE, dtype=torch.float16)
up = torch.full((m, inter), 2**-10, device=DEVICE, dtype=torch.float16)
up[:, :64:32] = 1.0
up[:, 64:] = 0.0
gate_up = torch.cat((gate, up), dim=-1)
got_q, got_s = swiglu_oai_quantize_mxfp8(gate_up, alpha=0.0, beta=0.0, limit=None)
ref_q, ref_s = _mxfp8_e4m3_quantize_torch(up, is_sf_swizzled_layout=False)
assert torch.equal(got_s, ref_s)
assert torch.equal(got_q, ref_q)
# --------------------------------------------------------------------------- #
# Fused MXFP8 activation quant (Triton vs torch reference)
# --------------------------------------------------------------------------- #
@pytest.mark.parametrize("shape", [(64, 4096), (1, 6144), (333, 2048)])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@torch.inference_mode()
def test_mxfp8_quant_triton_matches_torch(shape, dtype):
torch.manual_seed(0)
x = torch.randn(*shape, device=DEVICE, dtype=dtype)
xq_t, s_t = _mxfp8_e4m3_quantize_torch(x, is_sf_swizzled_layout=False)
xq_k, s_k = _mxfp8_e4m3_quantize_triton(x)
assert s_k.shape == s_t.shape == (shape[0], shape[1] // 32)
# Both paths use the E4M3-aware scale calculation; allow a 1-step
# difference at exact powers of two due to floating-point rounding.
assert (s_k.int() - s_t.int()).abs().max().item() <= 1
# Dequantized values agree to fp8 granularity.
deq_t = dequant_mxfp8_to_bf16(xq_t, s_t)
deq_k = dequant_mxfp8_to_bf16(xq_k, s_k)
assert _relerr(deq_k, deq_t) < 1e-2
# --------------------------------------------------------------------------- #
# Native MXFP8 linear (dot_scaled) vs dequant-to-bf16 matmul
# --------------------------------------------------------------------------- #
@requires_gfx950
@pytest.mark.parametrize("m,n,k", [(64, 256, 128), (37, 512, 256), (1, 6144, 4096)])
@torch.inference_mode()
def test_mxfp8_native_linear(m, n, k):
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
_mxfp8_dot_scaled_linear,
)
torch.manual_seed(0)
w_bf16 = torch.randn(n, k, device=DEVICE, dtype=torch.bfloat16) * 0.1
w_fp8, w_scale = _mxfp8_e4m3_quantize_torch(w_bf16, is_sf_swizzled_layout=False)
x = torch.randn(m, k, device=DEVICE, dtype=torch.bfloat16) * 0.5
got = _mxfp8_dot_scaled_linear(x, w_fp8, w_scale)
# Reference: consume the SAME quantized weights (isolates activation-quant
# noise) -> dequant to bf16, plain matmul.
w_deq = dequant_mxfp8_to_bf16(w_fp8, w_scale)
ref = torch.nn.functional.linear(x, w_deq).to(x.dtype)
assert got.shape == (m, n)
# Only the activation is re-quantized inside the kernel -> small MX noise.
assert _relerr(got, ref) < 5e-2
# --------------------------------------------------------------------------- #
# Native MXFP8 MoE (dot_scaled grouped GEMM) vs dequant-to-bf16 MoE math
# --------------------------------------------------------------------------- #
def _ref_moe(x, w13, w2, topk_weights, topk_ids, alpha, beta, limit):
T, H = x.shape
inter = w2.shape[-1]
top_k = topk_ids.shape[1]
out = torch.zeros(T, H, device=x.device, dtype=torch.float32)
for t in range(T):
for j in range(top_k):
e = int(topk_ids[t, j].item())
g1 = x[t].float() @ w13[e].float().T # [2I]
gate = g1[:inter]
up = g1[inter:]
if limit is not None:
gate = gate.clamp(max=limit)
up = up.clamp(min=-limit, max=limit)
act = gate * torch.sigmoid(alpha * gate) * (up + beta)
g2 = act @ w2[e].float().T # [H]
out[t] += topk_weights[t, j].float() * g2
return out.to(x.dtype)
@requires_gfx950
@pytest.mark.parametrize(
"T,H,inter,E,top_k", [(8, 256, 512, 8, 2), (1, 512, 256, 16, 4)]
)
@torch.inference_mode()
def test_mxfp8_native_moe(T, H, inter, E, top_k):
from vllm.model_executor.layers.fused_moe.experts.mxfp8_native_moe import (
fused_moe_mxfp8_native,
)
torch.manual_seed(0)
alpha, beta, limit = 1.702, 1.0, 7.0
w13_bf16 = torch.randn(E, 2 * inter, H, device=DEVICE, dtype=torch.bfloat16) * 0.1
w2_bf16 = torch.randn(E, H, inter, device=DEVICE, dtype=torch.bfloat16) * 0.1
w13_fp8, w13_scale = _mxfp8_e4m3_quantize_torch(
w13_bf16, is_sf_swizzled_layout=False
)
w2_fp8, w2_scale = _mxfp8_e4m3_quantize_torch(w2_bf16, is_sf_swizzled_layout=False)
x = torch.randn(T, H, device=DEVICE, dtype=torch.bfloat16) * 0.5
logits = torch.randn(T, E, device=DEVICE, dtype=torch.float32)
topk_weights, topk_ids = logits.softmax(dim=-1).topk(top_k, dim=-1)
topk_weights = topk_weights.to(torch.float32)
topk_ids = topk_ids.to(torch.int32)
got = fused_moe_mxfp8_native(
x,
w13_fp8,
w13_scale,
w2_fp8,
w2_scale,
topk_weights,
topk_ids,
alpha=alpha,
beta=beta,
limit=limit,
global_num_experts=E,
expert_map=None,
)
# Reference consumes the dequantized weights (same bits the kernel reads).
w13_deq = dequant_mxfp8_to_bf16(w13_fp8, w13_scale)
w2_deq = dequant_mxfp8_to_bf16(w2_fp8, w2_scale)
ref = _ref_moe(x, w13_deq, w2_deq, topk_weights, topk_ids, alpha, beta, limit)
assert got.shape == (T, H)
assert _relerr(got, ref) < 5e-2
# --------------------------------------------------------------------------- #
# Native MXFP8 grouped GEMM (dot_scaled) vs pure-PyTorch grouped matmul
# --------------------------------------------------------------------------- #
def _ref_grouped_gemm(a_deq, w_deq, topk_ids, a_div, num_valid, mul_weight=None):
"""Pure-PyTorch reference for ``_grouped_gemm_mxfp8``.
For each routed (expanded) token ``tid in [0, num_valid)`` the kernel writes
``out[tid] = a[tid // a_div] @ w[expert(tid)].T`` (fp32 accumulate), optionally
scaled by ``mul_weight[tid]``. The expert for ``tid`` is ``topk_ids.flatten()
[tid]`` (row-major expansion: ``tid = token*top_k + slot``). This is computed
here with plain ``torch.matmul`` on the dequantized operands — independent of
the Triton ``dot_scaled`` path and of the (separate) aiter backend.
"""
eids = topk_ids.reshape(-1)
n = w_deq.shape[1]
out = torch.empty(num_valid, n, dtype=torch.float32, device=a_deq.device)
for tid in range(num_valid):
e = int(eids[tid].item())
out[tid] = a_deq[tid // a_div].float() @ w_deq[e].float().T
if mul_weight is not None:
out[tid] *= float(mul_weight[tid].item())
return out
@requires_gfx950
@pytest.mark.parametrize("T,N,K,E,top_k", [(8, 256, 128, 8, 2), (5, 512, 256, 16, 4)])
@pytest.mark.parametrize("weighted", [False, True])
@torch.inference_mode()
def test_mxfp8_grouped_gemm_native(T, N, K, E, top_k, weighted):
"""Directly exercise ``_grouped_gemm_mxfp8`` against a non-Triton reference.
Covers both call modes used by ``fused_moe_mxfp8_native``:
* ``weighted=False`` -> g1: ``a_div=top_k`` (a-row shared across the top_k
expansions of a token), no per-token weight.
* ``weighted=True`` -> g2: ``a_div=1`` (one a-row per expansion), output
scaled by ``topk_weights``.
"""
from vllm.model_executor.layers.fused_moe.experts.mxfp8_native_moe import (
_grouped_gemm_mxfp8,
)
from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
moe_align_block_size,
)
torch.manual_seed(0)
block_m = 64
a_div = 1 if weighted else top_k
m_routed = T * top_k
# a-rows: g1 reads one row per token (a_div=top_k); g2 one per expansion.
a_rows = m_routed if weighted else T
a_bf16 = torch.randn(a_rows, K, device=DEVICE, dtype=torch.bfloat16) * 0.5
w_bf16 = torch.randn(E, N, K, device=DEVICE, dtype=torch.bfloat16) * 0.1
a_fp8, a_scale = _mxfp8_e4m3_quantize_torch(a_bf16, is_sf_swizzled_layout=False)
w_fp8, w_scale = _mxfp8_e4m3_quantize_torch(w_bf16, is_sf_swizzled_layout=False)
logits = torch.randn(T, E, device=DEVICE, dtype=torch.float32)
topk_weights, topk_ids = logits.softmax(dim=-1).topk(top_k, dim=-1)
topk_weights = topk_weights.to(torch.float32)
topk_ids = topk_ids.to(torch.int32)
mul = topk_weights.reshape(-1) if weighted else None
sorted_ids, expert_ids, num_post = moe_align_block_size(
topk_ids, block_m, E, None, ignore_invalid_experts=False
)
got = _grouped_gemm_mxfp8(
a_fp8,
a_scale,
w_fp8,
w_scale,
sorted_ids,
expert_ids,
num_post,
m_routed,
top_k,
block_m,
torch.bfloat16,
a_div=a_div,
mul_weight_by=mul,
)
# Reference: dequant the SAME bits the kernel reads, plain torch matmul.
a_deq = dequant_mxfp8_to_bf16(a_fp8, a_scale)
w_deq = dequant_mxfp8_to_bf16(w_fp8, w_scale)
ref = _ref_grouped_gemm(a_deq, w_deq, topk_ids, a_div, m_routed, mul)
assert got.shape == (m_routed, N)
assert _relerr(got, ref) < 5e-2
# --------------------------------------------------------------------------- #
# MXFP8 linear emulation: BF16-at-load (default) vs per-step dequant + switch
# --------------------------------------------------------------------------- #
@pytest.mark.parametrize("shape", [(512, 2048), (1, 6144)])
@pytest.mark.parametrize("act_dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("dequant_at_load", [True, False])
@torch.inference_mode()
def test_mxfp8_linear_emulation_bf16_at_load(
shape, act_dtype, dequant_at_load, monkeypatch
):
"""EmulationMxfp8LinearKernel load-time BF16 dequant (default) and the
``VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD=0`` per-step fallback must produce the
same result; the dtype-match (BF16/FP16 activations) must also hold."""
from vllm.model_executor.kernels.linear.mxfp8.emulation import (
EmulationMxfp8LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp8.Mxfp8LinearKernel import (
Mxfp8LinearLayerConfig,
)
monkeypatch.setenv(
"VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD", "1" if dequant_at_load else "0"
)
N, K = shape
torch.manual_seed(0)
w_bf16 = torch.randn(N, K, device=DEVICE, dtype=torch.bfloat16)
w_fp8, w_scale = _mxfp8_e4m3_quantize_torch(w_bf16, is_sf_swizzled_layout=False)
assert w_scale.shape == (N, K // 32)
# Reference: dequant once, plain linear in the activation dtype.
w_ref = dequant_mxfp8_to_bf16(w_fp8, w_scale).to(act_dtype)
x = torch.randn(7, K, device=DEVICE, dtype=act_dtype)
out_ref = torch.nn.functional.linear(x, w_ref)
layer = torch.nn.Module()
layer.weight = torch.nn.Parameter(w_fp8.clone(), requires_grad=False)
layer.weight_scale = torch.nn.Parameter(w_scale.clone(), requires_grad=False)
kernel = EmulationMxfp8LinearKernel(Mxfp8LinearLayerConfig(weight_shape=(N, K)))
kernel.process_weights_after_loading(layer)
if dequant_at_load:
# weights converted to BF16 at load (>= 2-byte)
assert layer.weight.element_size() >= 2
else:
# opt-out: weights stay 1-byte MXFP8, dequant happens per-step
assert layer.weight.element_size() == 1
out = kernel.apply_weights(layer, x)
assert out.dtype == act_dtype # dtype-match preserved (no tl.dot/F.linear crash)
assert _relerr(out.float(), out_ref.float()) < 2e-2
# --------------------------------------------------------------------------- #
# Native MXFP8 linear: the K % 128 != 0 weight is dequantized once at load
# --------------------------------------------------------------------------- #
@requires_gfx950
# (5120, 576) and (5120, 288): DSv4.1 shared-expert down_proj at TP4 and TP8.
@pytest.mark.parametrize("shape", [(5120, 576), (5120, 288), (256, 2048)])
@torch.inference_mode()
def test_mxfp8_rocm_native_unaligned_k_dequantizes_at_load(shape):
"""``dot_scaled`` tiles K by 128, so an unaligned weight must not reach it.
``process_weights_after_loading`` converts those to BF16 once; a weight that
never went through it has to fall back per step rather than take the
invalid path, which is what keying the dispatch on the weight dtype alone
would do.
"""
from vllm.model_executor.kernels.linear.mxfp8 import rocm_native
from vllm.model_executor.kernels.linear.mxfp8.Mxfp8LinearKernel import (
Mxfp8LinearLayerConfig,
)
N, K = shape
aligned = K % 128 == 0
torch.manual_seed(0)
w_bf16 = torch.randn(N, K, device=DEVICE, dtype=torch.bfloat16) * 0.1
w_fp8, w_scale = _mxfp8_e4m3_quantize_torch(w_bf16, is_sf_swizzled_layout=False)
x = torch.randn(7, K, device=DEVICE, dtype=torch.bfloat16) * 0.5
out_ref = torch.nn.functional.linear(x, dequant_mxfp8_to_bf16(w_fp8, w_scale))
# dot_scaled re-quantizes the activation, the dequant paths do not.
tol = 5e-2 if aligned else 2e-3
def make_layer():
layer = torch.nn.Module()
layer.weight = torch.nn.Parameter(w_fp8.clone(), requires_grad=False)
layer.weight_scale = torch.nn.Parameter(w_scale.clone(), requires_grad=False)
return layer
kernel = rocm_native.RocmDotScaledMxfp8LinearKernel(
Mxfp8LinearLayerConfig(weight_shape=(N, K))
)
layer = make_layer()
kernel.process_weights_after_loading(layer)
assert (layer.weight.element_size() >= 2) is not aligned
assert _relerr(kernel.apply_weights(layer, x).float(), out_ref.float()) < tol
# An unaligned weight that is still FP8 must not reach dot_scaled.
unprocessed = make_layer()
unprocessed.weight_scale = torch.nn.Parameter(
w_scale[:N, : K // 32].contiguous(), requires_grad=False
)
calls = []
real = rocm_native._mxfp8_dot_scaled_linear
def _spy(*a, **k):
calls.append(1)
return real(*a, **k)
rocm_native._mxfp8_dot_scaled_linear = _spy
try:
out = kernel.apply_weights(unprocessed, x)
finally:
rocm_native._mxfp8_dot_scaled_linear = real
assert bool(calls) is aligned
assert _relerr(out.float(), out_ref.float()) < tol
# ── EP expert_mask handling for the FlyDSL (AITER_MXFP8) MoE ────────────────
# The map/mask choice lives in ``RoutedExperts.expert_map``: it hands AITER
# experts (``consumes_expert_mask``) the precomputed 0/1 ``expert_mask`` and
# everyone else the canonical -1 index map, keyed on the resolved experts kernel
# rather than the global VLLM_ROCM_USE_AITER switch. ``AiterMxfp8Experts.apply``
# forwards that mask to aiter unchanged (it no longer rebuilds it via
# ``(expert_map >= 0)``, which collapsed an already-0/1 mask to all-ones and
# produced EP garbage).
def _capture_expert_mask(expert_mask, *, global_num_experts):
"""Drive the real ``AiterMxfp8Experts.apply`` mask branch and capture the
``expert_mask`` it forwards to ``rocm_aiter_ops.fused_moe``."""
from types import SimpleNamespace
from unittest import mock
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp8_moe import (
AiterMxfp8Experts,
)
experts = object.__new__(AiterMxfp8Experts) # bypass heavy __init__
experts.quant_config = SimpleNamespace(gemm1_clamp_limit=None)
experts.w1_scale_val = None
experts.w2_scale_val = None
captured = {}
def _fake_fused_moe(hidden_states, w1, w2, tw, ti, *, expert_mask, **kw):
captured["expert_mask"] = expert_mask
return torch.zeros_like(hidden_states)
w1 = torch.zeros(1, device=DEVICE)
w2 = torch.zeros(1, device=DEVICE)
out = torch.zeros(4, 8, device=DEVICE, dtype=torch.bfloat16)
hidden = torch.zeros(4, 8, device=DEVICE, dtype=torch.bfloat16)
tw = torch.ones(4, 2, device=DEVICE)
ti = torch.zeros(4, 2, dtype=torch.int32, device=DEVICE)
with mock.patch.object(rocm_aiter_ops, "fused_moe", side_effect=_fake_fused_moe):
experts.apply(
output=out,
hidden_states=hidden,
w1=w1,
w2=w2,
topk_weights=tw,
topk_ids=ti,
activation=None,
global_num_experts=global_num_experts,
expert_map=expert_mask,
a1q_scale=None,
a2_scale=None,
workspace13=None,
workspace2=None,
expert_tokens_meta=None,
apply_router_weight_on_input=False,
)
return captured["expert_mask"]
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
def test_aiter_mxfp8_apply_forwards_expert_mask_unchanged():
"""AiterMxfp8Experts.apply forwards the precomputed 0/1 mask to aiter as-is
(no re-derivation that would collapse an already-0/1 mask to all-ones)."""
# 0/1 local-expert mask over global ids + trailing sentinel (rank owns 0..3).
ep_mask = torch.tensor(
[1, 1, 1, 1, 0, 0, 0, 0, 0], dtype=torch.int32, device=DEVICE
)
got = _capture_expert_mask(ep_mask, global_num_experts=8)
assert torch.equal(got.cpu().to(torch.int32), ep_mask.cpu().to(torch.int32))
assert got.sum().item() == 4 # the 4 local experts, not all 9
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
def test_routed_experts_expert_map_delegates_to_kernel():
"""RoutedExperts.expert_map returns the 0/1 mask only for kernels that set
``consumes_expert_mask`` (AITER), and the canonical -1 map otherwise -- keyed
on the resolved kernel, not the global aiter switch."""
from types import SimpleNamespace
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
canonical = torch.tensor([0, 1, 2, 3, -1, -1, -1, -1], dtype=torch.int32)
mask = torch.tensor([1, 1, 1, 1, 0, 0, 0, 0, 0], dtype=torch.int32)
def resolve(consumes_mask, *, has_moe_kernel=True):
moe_kernel = (
SimpleNamespace(
fused_experts=SimpleNamespace(consumes_expert_mask=consumes_mask)
)
if has_moe_kernel
else None
)
layer = SimpleNamespace(
_expert_map=canonical,
expert_mask=mask,
quant_method=SimpleNamespace(moe_kernel=moe_kernel),
)
return RoutedExperts.expert_map.fget(layer)
assert torch.equal(resolve(True), mask) # AITER kernel -> 0/1 mask
assert torch.equal(resolve(False), canonical) # non-AITER -> canonical map
assert torch.equal(resolve(False, has_moe_kernel=False), canonical) # non-modular
# --------------------------------------------------------------------------- #
# Strided K/V operand for the AITER sparse-PA cache insert
# --------------------------------------------------------------------------- #
def _fused_qkv_kv_slices(num_tokens, num_kv_heads, head_dim, num_q_heads=16):
"""K/V as the sparse-PA insert sees them: column slices of a fused qkv."""
kv = num_kv_heads * head_dim
row = num_q_heads * head_dim + 2 * kv + 2 * head_dim
qkv = torch.randn((num_tokens, row), dtype=torch.bfloat16, device=DEVICE)
k_start = num_q_heads * head_dim
k = qkv[:, k_start : k_start + kv].view(num_tokens, num_kv_heads, head_dim)
v = qkv[:, k_start + kv : k_start + 2 * kv].view(num_tokens, num_kv_heads, head_dim)
return k, v
def _asm_kv_cache(pages, num_kv_heads, head_dim):
"""Page-16 K/V cache views matching ``_ensure_aiter_sparse_pa_kv_cache``."""
x = 16 // torch.tensor([], dtype=torch.bfloat16).element_size()
key = torch.zeros(
(pages, num_kv_heads, head_dim // x, 16, x),
dtype=torch.bfloat16,
device=DEVICE,
)
value = torch.zeros(
(pages, num_kv_heads, 16 // x, head_dim, x),
dtype=torch.bfloat16,
device=DEVICE,
)
return key, value
@pytest.mark.parametrize("num_tokens", [1, 8, 129, 1024])
@pytest.mark.parametrize("num_kv_heads", [1, 2])
def test_kv_insert_operand_matches_contiguous(num_tokens, num_kv_heads):
"""A strided K/V slice must insert bit-identically to a contiguous copy."""
reshape_and_cache = pytest.importorskip("aiter").reshape_and_cache
head_dim = 128
pages = num_tokens // 16 + 4
slots = torch.randperm(pages * 16, device=DEVICE)[:num_tokens].to(torch.int32)
torch.manual_seed(num_tokens)
k, v = _fused_qkv_kv_slices(num_tokens, num_kv_heads, head_dim)
assert num_tokens == 1 or not k.is_contiguous()
caches = []
for operand in (lambda t: t.contiguous(), _kv_insert_operand):
key_cache, value_cache = _asm_kv_cache(pages, num_kv_heads, head_dim)
reshape_and_cache(
operand(k),
operand(v),
key_cache,
value_cache,
slots,
kv_cache_dtype="auto",
asm_layout=True,
)
caches.append((key_cache, value_cache))
(ref_k, ref_v), (got_k, got_v) = caches
assert torch.equal(ref_k, got_k)
assert torch.equal(ref_v, got_v)
def test_kv_insert_operand_copies_unsupported_layouts():
"""Only a row-strided slice passes through; anything else is copied."""
k, _ = _fused_qkv_kv_slices(32, 2, 128)
assert _kv_insert_operand(k) is k
inner_strided = k[:, :, ::2]
assert _kv_insert_operand(inner_strided).is_contiguous()
flat = k.reshape(32, -1)
assert _kv_insert_operand(flat).is_contiguous()
@@ -1,189 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.platforms import current_platform
if not torch.cuda.is_available() or not current_platform.supports_fp8():
pytest.skip(
"MiniMax-M3 FP8 sparse attention scale tests require an FP8 GPU.",
allow_module_level=True,
)
if current_platform.is_rocm():
from vllm.models.minimax_m3.amd.ops.sparse_attn import (
SPARSE_BLOCK_SIZE,
minimax_m3_sparse_attn,
minimax_m3_sparse_attn_decode,
)
else:
from vllm.models.minimax_m3.common.ops.sparse_attn import (
SPARSE_BLOCK_SIZE,
minimax_m3_sparse_attn,
minimax_m3_sparse_attn_decode,
)
DEVICE = "cuda"
DTYPE = torch.bfloat16
HEAD_DIM = 128
NUM_KV_HEADS = 1
NUM_HEADS = 2
K_SCALE = 0.25
V_SCALE = 0.5
def _scale_tensors(mode: str, num_blocks: int):
if mode == "scalar":
k_scale = torch.tensor(K_SCALE, dtype=torch.float32, device=DEVICE)
v_scale = torch.tensor(V_SCALE, dtype=torch.float32, device=DEVICE)
else:
shape = (NUM_KV_HEADS, num_blocks * SPARSE_BLOCK_SIZE)
k_scale = torch.full(shape, K_SCALE, dtype=torch.float32, device=DEVICE)
v_scale = torch.full(shape, V_SCALE, dtype=torch.float32, device=DEVICE)
return k_scale, v_scale
def _make_kv_cache(num_blocks: int, seed: int):
torch.manual_seed(seed)
fp8_dtype = current_platform.fp8_dtype()
kv_ref = torch.randn(
num_blocks,
NUM_KV_HEADS,
SPARSE_BLOCK_SIZE,
2 * HEAD_DIM,
dtype=DTYPE,
device=DEVICE,
)
kv_fp8 = torch.empty_like(kv_ref, dtype=fp8_dtype)
kv_fp8[..., :HEAD_DIM] = (kv_ref[..., :HEAD_DIM].float() / K_SCALE).to(fp8_dtype)
kv_fp8[..., HEAD_DIM:] = (kv_ref[..., HEAD_DIM:].float() / V_SCALE).to(fp8_dtype)
kv_dequant = torch.empty_like(kv_ref)
kv_dequant[..., :HEAD_DIM] = (kv_fp8[..., :HEAD_DIM].float() * K_SCALE).to(DTYPE)
kv_dequant[..., HEAD_DIM:] = (kv_fp8[..., HEAD_DIM:].float() * V_SCALE).to(DTYPE)
return kv_fp8, kv_dequant
@pytest.mark.parametrize("scale_mode", ["scalar", "per_token_head"])
@torch.inference_mode()
def test_minimax_m3_sparse_prefill_fp8_kv_scales(scale_mode: str):
total_q = 17
num_blocks = 1
torch.manual_seed(0)
q = torch.randn(total_q, NUM_HEADS, HEAD_DIM, dtype=DTYPE, device=DEVICE) * 0.1
kv_fp8, kv_dequant = _make_kv_cache(num_blocks, seed=1)
k_scale, v_scale = _scale_tensors(scale_mode, num_blocks)
topk = torch.zeros(NUM_KV_HEADS, total_q, 1, dtype=torch.int32, device=DEVICE)
block_table = torch.zeros(1, 1, dtype=torch.int32, device=DEVICE)
cu_seqlens = torch.tensor([0, total_q], dtype=torch.int32, device=DEVICE)
seq_lens = torch.tensor([total_q], dtype=torch.int32, device=DEVICE)
prefix_lens = torch.zeros(1, dtype=torch.int32, device=DEVICE)
got = torch.empty_like(q)
ref = torch.empty_like(q)
unscaled = torch.empty_like(q)
minimax_m3_sparse_attn(
q,
kv_fp8,
topk,
block_table,
cu_seqlens,
seq_lens,
prefix_lens,
total_q,
NUM_KV_HEADS,
HEAD_DIM**-0.5,
got,
k_scale=k_scale,
v_scale=v_scale,
)
minimax_m3_sparse_attn(
q,
kv_dequant,
topk,
block_table,
cu_seqlens,
seq_lens,
prefix_lens,
total_q,
NUM_KV_HEADS,
HEAD_DIM**-0.5,
ref,
)
minimax_m3_sparse_attn(
q,
kv_fp8,
topk,
block_table,
cu_seqlens,
seq_lens,
prefix_lens,
total_q,
NUM_KV_HEADS,
HEAD_DIM**-0.5,
unscaled,
)
torch.testing.assert_close(got, ref, rtol=2e-2, atol=2e-2)
assert not torch.allclose(unscaled, ref, rtol=1e-1, atol=1e-1)
@pytest.mark.parametrize("scale_mode", ["scalar", "per_token_head"])
@torch.inference_mode()
def test_minimax_m3_sparse_decode_fp8_kv_scales(scale_mode: str):
total_q = 2
num_blocks = 1
torch.manual_seed(2)
q = torch.randn(total_q, NUM_HEADS, HEAD_DIM, dtype=DTYPE, device=DEVICE) * 0.1
kv_fp8, kv_dequant = _make_kv_cache(num_blocks, seed=3)
k_scale, v_scale = _scale_tensors(scale_mode, num_blocks)
topk = torch.zeros(NUM_KV_HEADS, total_q, 1, dtype=torch.int32, device=DEVICE)
block_table = torch.zeros(total_q, 1, dtype=torch.int32, device=DEVICE)
seq_lens = torch.tensor([64, 128], dtype=torch.int32, device=DEVICE)
got = torch.empty_like(q)
ref = torch.empty_like(q)
unscaled = torch.empty_like(q)
minimax_m3_sparse_attn_decode(
q,
kv_fp8,
topk,
block_table,
seq_lens,
NUM_KV_HEADS,
HEAD_DIM**-0.5,
got,
decode_query_len=1,
k_scale=k_scale,
v_scale=v_scale,
)
minimax_m3_sparse_attn_decode(
q,
kv_dequant,
topk,
block_table,
seq_lens,
NUM_KV_HEADS,
HEAD_DIM**-0.5,
ref,
decode_query_len=1,
)
minimax_m3_sparse_attn_decode(
q,
kv_fp8,
topk,
block_table,
seq_lens,
NUM_KV_HEADS,
HEAD_DIM**-0.5,
unscaled,
decode_query_len=1,
)
torch.testing.assert_close(got, ref, rtol=2e-2, atol=2e-2)
assert not torch.allclose(unscaled, ref, rtol=1e-1, atol=1e-1)
File diff suppressed because it is too large Load Diff
-36
View File
@@ -1,36 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU-only integration tests for the UltraQuant cache dtype."""
from vllm.v1.attention.backends.ultraquant_attn import (
UltraQuantAttentionBackend,
)
from vllm.v1.attention.ops.flydsl_ultraquant_decode import (
ultraquant_flydsl_decode_eligible,
)
def test_ultraquant_prefers_block_size_64():
# FlyDSL D=256 decode geometry requires a 64-token KV block.
assert UltraQuantAttentionBackend.get_preferred_block_size(16) == 64
assert UltraQuantAttentionBackend.get_preferred_block_size(128) == 64
def test_ultraquant_flydsl_decode_routing():
"""CPU check of FlyDSL vs unified-Triton decode eligibility."""
eligible = dict(
head_size=256,
num_kv_groups=8,
has_sinks=False,
sliding_window=None,
flydsl_loaded=True,
)
assert ultraquant_flydsl_decode_eligible(**eligible)
for gqa in (6, 8, 16):
assert ultraquant_flydsl_decode_eligible(**{**eligible, "num_kv_groups": gqa})
assert not ultraquant_flydsl_decode_eligible(**{**eligible, "flydsl_loaded": False})
assert not ultraquant_flydsl_decode_eligible(**{**eligible, "head_size": 128})
assert not ultraquant_flydsl_decode_eligible(**{**eligible, "num_kv_groups": 4})
assert not ultraquant_flydsl_decode_eligible(**{**eligible, "has_sinks": True})
assert not ultraquant_flydsl_decode_eligible(**{**eligible, "sliding_window": 4096})
@@ -1,106 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""UltraQuant inherits TurboQuant's forward/do_kv_cache_update orchestration.
UltraQuant only overrides the store/decode/prefill seams, so a TurboQuant-side
change to a seam signature, to an inherited helper, or to the ``layer._tq_*``
attributes ``forward`` reads will break UltraQuant at serve time without
touching any UltraQuant kernel test. These checks pin that contract.
"""
import ast
import inspect
import textwrap
from vllm.v1.attention.backends import ultraquant_attn
from vllm.v1.attention.backends.turboquant_attn import TurboQuantAttentionImpl
from vllm.v1.attention.backends.ultraquant_attn import UltraQuantAttentionImpl
_SEAMS = ("_ensure_on_device", "_store_kv", "_decode_attention", "_prefill_attention")
def _method_ast(cls: type, name: str) -> ast.AST:
return ast.parse(textwrap.dedent(inspect.getsource(getattr(cls, name))))
def _tq_attrs_read(cls: type, name: str) -> set[str]:
"""``_tq_*`` attribute names read (not assigned) in ``cls.name``."""
tree = _method_ast(cls, name)
assigned = {
t.attr
for node in ast.walk(tree)
if isinstance(node, ast.Assign)
for t in node.targets
if isinstance(t, ast.Attribute)
}
read = {
node.attr
for node in ast.walk(tree)
if isinstance(node, ast.Attribute) and node.attr.startswith("_tq_")
}
return read - assigned
def _tq_attrs_written(cls: type, name: str) -> set[str]:
tree = _method_ast(cls, name)
return {
t.attr
for node in ast.walk(tree)
if isinstance(node, ast.Assign)
for t in node.targets
if isinstance(t, ast.Attribute) and t.attr.startswith("_tq_")
}
def test_ultraquant_matches_turboquant_seam_contract():
# 1. Each overridden seam accepts every parameter the base declares, so any
# call the inherited orchestration makes stays valid for UltraQuant.
for name in _SEAMS:
base = list(
inspect.signature(getattr(TurboQuantAttentionImpl, name)).parameters
)
override = list(
inspect.signature(getattr(UltraQuantAttentionImpl, name)).parameters
)
assert override[: len(base)] == base, (
f"UltraQuantAttentionImpl.{name} no longer accepts the arguments "
f"TurboQuantAttentionImpl passes: base={base} override={override}"
)
# 2. Keyword arguments UltraQuant passes to *inherited* TurboQuant helpers
# must exist on those helpers (e.g. _flash_attn_varlen).
own = vars(UltraQuantAttentionImpl)
module = ast.parse(inspect.getsource(ultraquant_attn))
for node in ast.walk(module):
if not isinstance(node, ast.Call):
continue
fn = node.func
if not (
isinstance(fn, ast.Attribute)
and isinstance(fn.value, ast.Name)
and fn.value.id == "self"
):
continue
target = getattr(TurboQuantAttentionImpl, fn.attr, None)
if target is None or fn.attr in own or not callable(target):
continue
params = inspect.signature(target).parameters
if any(p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values()):
continue
for kw in node.keywords:
if kw.arg is None:
continue
assert kw.arg in params, (
f"UltraQuant calls inherited {fn.attr}(..., {kw.arg}=...) but "
f"TurboQuantAttentionImpl.{fn.attr} accepts {list(params)}"
)
# 3. Every layer._tq_* attribute the inherited forward reads is set by
# UltraQuant's _ensure_on_device.
required = _tq_attrs_read(TurboQuantAttentionImpl, "forward")
provided = _tq_attrs_written(UltraQuantAttentionImpl, "_ensure_on_device")
assert required, "expected inherited forward to read layer._tq_* attributes"
assert required <= provided, (
f"UltraQuant._ensure_on_device does not set {sorted(required - provided)}, "
f"which the inherited TurboQuant forward reads"
)