[Attention] Re-enable cross-layer KV cache layout for MLA via stride-aware kernels (#45111)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Yifan Qiao
2026-06-22 06:57:02 -07:00
committed by GitHub
parent a4610da0c6
commit aa4990a9a2
13 changed files with 847 additions and 21 deletions
@@ -136,8 +136,12 @@ typename T::Fmha::Arguments args_from_options(
StrideQ stride_Q_pe = cute::make_tuple(
static_cast<int64_t>(q_pe.stride(1)), _1{}, static_cast<int64_t>(q_pe.stride(0)));
// Read the token and page strides from the cache tensor instead of assuming
// packed pages, so strided views (e.g. per-layer views into a cross-layer
// block-major cache) are addressed correctly.
StrideK stride_C = cute::make_tuple(
static_cast<int64_t>(0 + D_latent + D_rope), _1{}, static_cast<int64_t>(page_size * (D_latent + D_rope)));
static_cast<int64_t>(kv_c_and_k_pe_cache.stride(1)), _1{},
static_cast<int64_t>(kv_c_and_k_pe_cache.stride(0)));
StrideLSE stride_PT = cute::make_stride(_1{}, page_count_per_seq);
StrideLSE stride_LSE = cute::make_tuple(_1{}, 0 + H);
StrideO stride_O = cute::make_tuple(static_cast<int64_t>(0 + D_latent), _1{}, static_cast<int64_t>(0 + H * D_latent));
+6 -7
View File
@@ -549,7 +549,7 @@ __global__ void indexer_k_quant_and_cache_kernel(
const int head_dim, // dimension of each head
const int quant_block_size, // quantization block size
const int cache_block_size, // cache block size
const int cache_stride, // stride for each token in kv_cache
const int64_t cache_block_stride, // stride for each block in kv_cache
const bool use_ue8m0 // use ue8m0 scale format
) {
@@ -590,16 +590,15 @@ __global__ void indexer_k_quant_and_cache_kernel(
scale = exp2f(ceilf(log2f(scale)));
}
const int64_t dst_offset = block_idx * cache_block_size * cache_stride +
block_offset * head_dim + head_dim_idx;
const int64_t dst_offset =
block_idx * cache_block_stride + block_offset * head_dim + head_dim_idx;
for (int i = 0; i < VEC_SIZE; i++) {
kv_cache[dst_offset + i] =
fp8::scaled_convert<cache_t, scalar_t, kv_dt>(k_val_ptr[i], scale);
}
if (threadIdx.x == 0) {
const int64_t dst_scale_idx =
block_idx * cache_block_size * cache_stride +
cache_block_size * head_dim +
block_idx * cache_block_stride + cache_block_size * head_dim +
(block_offset * head_dim + head_dim_idx) * 4 / quant_block_size;
reinterpret_cast<float*>(kv_cache)[dst_scale_idx / 4] = scale;
}
@@ -1452,7 +1451,7 @@ void cp_gather_and_upconvert_fp8_kv_cache(
reinterpret_cast<KV_T*>(k.data_ptr()), \
reinterpret_cast<CACHE_T*>(kv_cache.data_ptr()), \
slot_mapping.const_data_ptr<int64_t>(), head_dim, quant_block_size, \
cache_block_size, cache_stride, use_ue8m0);
cache_block_size, cache_block_stride, use_ue8m0);
void indexer_k_quant_and_cache(
torch::stable::Tensor& k, // [num_tokens, head_dim]
@@ -1463,7 +1462,7 @@ void indexer_k_quant_and_cache(
int num_tokens = k.size(0);
int head_dim = k.size(1);
int cache_block_size = kv_cache.size(1);
int cache_stride = kv_cache.size(2);
int64_t cache_block_stride = kv_cache.stride(0);
bool use_ue8m0 = scale_fmt == "ue8m0";
STD_TORCH_CHECK(k.device() == kv_cache.device(),
@@ -212,3 +212,69 @@ def test_cutlass_mla_decode(
print(
f"{t:.3f} ms, {FLOPS / 10**9 / t:.0f} TFLOPS,", f"{bytes / 10**6 / t:.0f} GB/s"
)
@pytest.mark.skipif(
not current_platform.has_device_capability(100),
reason=CUTLASS_MLA_UNSUPPORTED_REASON,
)
@torch.inference_mode()
def test_cutlass_mla_decode_cross_layer_view():
"""The kernel must read the cache's page-dim stride instead of assuming
pages are packed back-to-back. A per-layer view into a cross-layer
(block-major) cache has stride(0) inflated by num_layers; outputs must
match a contiguous cache holding the same data exactly."""
device = torch.device("cuda:0")
torch.set_default_dtype(torch.bfloat16)
torch.set_default_device(device)
torch.manual_seed(42)
b, mean_sk, d, dv, block_size = 4, 512, 576, 512, 64
num_layers, layer_idx = 3, 1
scale = math.sqrt(d) ** (-1)
num_pages = b * (mean_sk // block_size)
cache_seqlens = torch.full((b,), mean_sk, dtype=torch.int32)
block_table = torch.arange(num_pages, dtype=torch.int32).view(
b, mean_sk // block_size
)
kv_contig = torch.randn(num_pages, block_size, d)
# Neighbor layers hold random data so packed-pages addressing reads
# garbage rather than zeros.
kv_cross_layer = torch.randn(num_pages, num_layers, block_size, d)
kv_view = kv_cross_layer[:, layer_idx]
kv_view.copy_(kv_contig)
assert kv_view.stride(0) == num_layers * block_size * d
q_nope = torch.randn(b, 128, dv)
q_pe = torch.randn(b, 128, d - dv)
sm_count = num_compute_units(device.index)
workspace_size = ops.sm100_cutlass_mla_get_workspace_size(
mean_sk, b, sm_count, num_kv_splits=1
)
workspace = torch.empty(workspace_size, dtype=torch.uint8)
def run(cache):
out = torch.empty(b, 128, dv)
lse = torch.empty(b, 128, dtype=torch.float32)
ops.sm100_cutlass_mla_decode(
out,
lse,
q_nope,
q_pe,
cache,
cache_seqlens,
block_table,
workspace,
scale,
1,
)
return out, lse
out_contig, lse_contig = run(kv_contig)
out_view, lse_view = run(kv_view)
# Same data and same compute order; only addressing differs.
assert torch.equal(out_contig, out_view)
assert torch.equal(lse_contig, lse_view)
@@ -0,0 +1,566 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Bit-exact kernel equivalence for MLA decode/write kernels on the
cross-layer (block-major) KV cache layout.
The cross-layer layout carves each layer's per-block page out of a single
unified slot, so the per-layer view has an inflated ``stride(0)`` (the full
unified slot) and a non-zero storage offset. These tests confirm the MLA
kernels behind the backends that opt in to the layout (FlashMLA dense,
FlashInfer MLA dense, FlashMLA fp8 sparse, plus the ``concat_and_cache_mla``
write) honor that strided view bit-identically to a contiguous per-layer
cache, and that writes do not bleed into neighbouring layers' segments.
"""
import pytest
import torch
pytestmark = pytest.mark.skipif(
not torch.cuda.is_available(), reason="MLA cache kernels require CUDA"
)
def test_concat_and_cache_mla_into_unified_slot_view():
"""concat_and_cache_mla must write correctly into a per-layer view whose
block stride is the full unified slot (block-major), with zero bleed into
the other layers' segments of the same slot."""
from vllm import _custom_ops as ops
torch.manual_seed(0)
dev = "cuda"
kv_lora_rank = 512
pe = 64
entry = kv_lora_rank + pe
page = 64
num_blocks = 32
ntok = 200
kv_c = torch.randn(ntok, kv_lora_rank, device=dev, dtype=torch.bfloat16)
k_pe = torch.randn(ntok, pe, device=dev, dtype=torch.bfloat16)
slot = torch.randperm(num_blocks * page, device=dev, dtype=torch.int64)[:ntok]
scale = torch.tensor(1.0, device=dev)
def write(cache):
ops.concat_and_cache_mla(kv_c, k_pe, cache, slot, "auto", scale)
# Contiguous per-layer reference: (num_blocks, page, entry).
ref = torch.zeros(num_blocks, page, entry, device=dev, dtype=torch.bfloat16)
write(ref)
# Unified slot holding three layer pages per block. Carve the middle
# layer's view (non-zero offset, block stride == full unified slot).
layer_page_elems = page * entry
n_layers = 3
unified_slot_elems = n_layers * layer_page_elems
big = torch.zeros(num_blocks, unified_slot_elems, device=dev, dtype=torch.bfloat16)
flat = big.view(-1)
offset = layer_page_elems # middle layer
view = torch.as_strided(
flat,
size=(num_blocks, page, entry),
stride=(unified_slot_elems, entry, 1),
storage_offset=offset,
)
assert not view.is_contiguous()
assert view.stride(0) == unified_slot_elems
write(view)
# Bit-exact equivalence and zero bleed into the neighbour segments.
max_diff = (ref.float() - view.float()).abs().max().item()
assert max_diff == 0.0, f"max|Δ| = {max_diff}"
neighbour_lo = torch.as_strided(
flat, (num_blocks, layer_page_elems), (unified_slot_elems, 1), 0
)
neighbour_hi = torch.as_strided(
flat,
(num_blocks, layer_page_elems),
(unified_slot_elems, 1),
2 * layer_page_elems,
)
assert neighbour_lo.abs().max().item() == 0.0
assert neighbour_hi.abs().max().item() == 0.0
def test_flashmla_dense_decode_unified_slot_view():
"""FlashMLA dense decode (FLASHMLA backend, e.g. Kimi-K2-style dense MLA
on Hopper) must read a unified-slot block-major view bit-identically to a
contiguous per-layer cache."""
import vllm.v1.attention.ops.flashmla as fm
ok, reason = fm.is_flashmla_dense_supported()
if not ok:
pytest.skip(reason)
torch.manual_seed(0)
dev = "cuda"
dt = torch.bfloat16
head_dim = 576
hdv = 512
h_q = 128
page = 64
num_blocks = 64
bs = 4
n_layers = 3
layer = 1
q = torch.randn(bs, 1, h_q, head_dim, device=dev, dtype=dt) * 0.1
kv_data = torch.randn(num_blocks, page, 1, head_dim, device=dev, dtype=dt) * 0.1
# (A) contiguous per-layer reference.
cache_contiguous = kv_data.clone().contiguous()
# (B) unified slot: view one layer -> inflated stride(0), non-zero offset.
unified = (
torch.randn(num_blocks, n_layers, page, 1, head_dim, device=dev, dtype=dt) * 0.1
)
unified[:, layer].copy_(kv_data)
cache_view = unified[:, layer]
assert not cache_view.is_contiguous()
assert cache_view.stride(0) == n_layers * page * 1 * head_dim
max_blk = num_blocks // bs
block_table = torch.arange(num_blocks, device=dev, dtype=torch.int32).view(
bs, max_blk
)
cache_seqlens = torch.full((bs,), max_blk * page, device=dev, dtype=torch.int32)
def run(kc):
meta, num_splits = fm.get_mla_metadata()
out, _ = fm.flash_mla_with_kvcache(
q=q,
k_cache=kc,
block_table=block_table,
cache_seqlens=cache_seqlens,
head_dim_v=hdv,
tile_scheduler_metadata=meta,
num_splits=num_splits,
softmax_scale=head_dim**-0.5,
causal=True,
)
return out.clone().float()
out_ref = run(cache_contiguous)
out_view = run(cache_view)
assert torch.isfinite(out_ref).all()
assert out_ref.abs().max().item() > 0.0
assert (out_ref - out_view).abs().max().item() == 0.0
def test_flashinfer_mla_dense_decode_unified_slot_view():
"""FlashInfer MLA dense decode must read a unified-slot block-major view
(inflated stride(0), non-zero storage offset) bit-identically to a
contiguous per-layer cache."""
try:
from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla
except ImportError:
pytest.skip("flashinfer is not available")
from vllm.platforms import current_platform
if not current_platform.is_device_capability_family(100):
pytest.skip("FlashInfer trtllm-gen MLA requires sm100")
torch.manual_seed(0)
dev = "cuda"
dt = torch.bfloat16
kv_lora_rank = 512
qk_rope_head_dim = 64
qk_nope_head_dim = 128
head_dim = kv_lora_rank + qk_rope_head_dim # 576
num_qo_heads = 128
page = 64
num_blocks = 64
bs = 4
n_layers = 3 # >1 so the per-layer view's block stride is inflated.
layer = 1
q = torch.randn(bs, 1, num_qo_heads, head_dim, device=dev, dtype=dt)
kv_data = torch.randn(num_blocks, 1, page, head_dim, device=dev, dtype=dt)
# (A) contiguous per-layer reference.
kv_contiguous = kv_data.clone().contiguous()
# (B) unified slot: block b of every layer packed together; view one layer
# -> stride(0) is n_layers x larger and storage offset is non-zero.
unified = torch.randn(num_blocks, n_layers, 1, page, head_dim, device=dev, dtype=dt)
unified[:, layer].copy_(kv_data)
kv_view = unified[:, layer]
assert not kv_view.is_contiguous()
assert kv_view.stride(0) == n_layers * 1 * page * head_dim
max_blk = num_blocks // bs
block_tables = torch.arange(num_blocks, device=dev, dtype=torch.int32).view(
bs, max_blk
)
seq_lens = torch.full((bs,), max_blk * page, device=dev, dtype=torch.int32)
ws = torch.empty(128 * 1024 * 1024, dtype=torch.int8, device=dev)
scale = head_dim**-0.5
def run(kv):
return trtllm_batch_decode_with_kv_cache_mla(
query=q,
kv_cache=kv,
workspace_buffer=ws,
qk_nope_head_dim=qk_nope_head_dim,
kv_lora_rank=kv_lora_rank,
qk_rope_head_dim=qk_rope_head_dim,
block_tables=block_tables,
seq_lens=seq_lens,
max_seq_len=int(seq_lens.max().item()),
bmm1_scale=scale,
bmm2_scale=1.0,
).clone()
out_ref = run(kv_contiguous).float()
out_view = run(kv_view).float()
assert torch.isfinite(out_ref).all()
assert (out_ref - out_view).abs().max().item() == 0.0
def test_flashmla_fp8_sparse_decode_unified_slot_view():
"""FlashMLA fp8 sparse decode (DeepSeek V3.2/V4 DSA path) must read a
unified-slot block-major view bit-identically to a contiguous fp8_ds_mla
cache, with finite nonzero output."""
import vllm.v1.attention.ops.flashmla as fm
ok, reason = fm.is_flashmla_sparse_supported()
if not ok:
pytest.skip(reason)
torch.manual_seed(0)
dev = "cuda"
entry = 656 # fp8_ds_mla bytes per token
page = 64
num_blocks = 32
h_q = 128
head_dim = 576
hdv = 512
batch = 2
topk = 128
n_layers = 3
layer = 1
q = torch.randn(batch, 1, h_q, head_dim, device=dev, dtype=torch.bfloat16) * 0.1
# Structurally valid fp8 ds_mla payload: 512B fp8 + 16B f32 scales + 128B
# bf16 rope (random bytes corrupt the scale region and yield NaNs).
nope = (torch.randn(num_blocks, page, 1, 512, device=dev) * 0.1).to(
torch.float8_e4m3fn
)
scales = torch.ones(num_blocks, page, 1, 4, device=dev, dtype=torch.float32)
rope = (torch.randn(num_blocks, page, 1, 64, device=dev) * 0.1).to(torch.bfloat16)
payload = torch.cat(
[
nope.view(torch.uint8).view(num_blocks, page, 1, 512),
scales.view(torch.uint8).view(num_blocks, page, 1, 16),
rope.view(torch.uint8).view(num_blocks, page, 1, 128),
],
dim=-1,
).contiguous()
assert payload.shape[-1] == entry and payload.dtype == torch.uint8
# (A) contiguous reference.
cache_contiguous = payload.clone().contiguous()
# (B) unified slot: view one layer -> inflated stride(0), non-zero offset.
unified = torch.randint(
0, 256, (num_blocks, n_layers, page, 1, entry), device=dev, dtype=torch.uint8
)
unified[:, layer].copy_(payload)
cache_view = unified[:, layer]
assert not cache_view.is_contiguous()
assert cache_view.stride(0) == n_layers * page * 1 * entry
# Sparse indices: each batch uses its own disjoint blocks.
blocks_per_batch = num_blocks // batch
idx = torch.full((batch, 1, topk), -1, device=dev, dtype=torch.int32)
for b in range(batch):
slots: list[int] = []
for blk in range(b * blocks_per_batch, (b + 1) * blocks_per_batch):
slots.extend(blk * page + off for off in range(page))
slots_t = torch.tensor(slots[:topk], device=dev, dtype=torch.int32)
idx[b, 0, : slots_t.numel()] = slots_t
def run(kc):
meta, num_splits = fm.get_mla_metadata()
out, _ = fm.flash_mla_with_kvcache(
q=q,
k_cache=kc,
block_table=None,
cache_seqlens=None,
head_dim_v=hdv,
tile_scheduler_metadata=meta,
is_fp8_kvcache=True,
indices=idx,
softmax_scale=head_dim**-0.5,
)
return out.clone().float()
out_ref = run(cache_contiguous)
out_view = run(cache_view)
assert torch.isfinite(out_ref).all()
assert out_ref.abs().max().item() > 0.0
assert (out_ref - out_view).abs().max().item() == 0.0
def test_indexer_k_quant_and_cache_into_unified_slot_view():
"""indexer_k_quant_and_cache (DeepSeek V3.2/V4 DSA indexer K write) must
write correctly into a per-layer view whose block stride is the full
unified slot, with zero bleed into the other layers' segments."""
from vllm import _custom_ops as ops
torch.manual_seed(0)
dev = "cuda"
head_dim = 128
quant_block_size = 128
block_size = 64
num_blocks = 16
ntok = 100
# Indexer cache layout per token: head_dim fp8 bytes followed by
# head_dim * 4 / quant_block_size scale bytes.
cache_stride = head_dim + head_dim * 4 // quant_block_size
k = torch.randn(ntok, head_dim, device=dev, dtype=torch.bfloat16)
slot = torch.randperm(num_blocks * block_size, device=dev, dtype=torch.int64)[:ntok]
def write(cache):
ops.indexer_k_quant_and_cache(k, cache, slot, quant_block_size, "ue8m0")
# Contiguous per-layer reference.
ref = torch.zeros(
num_blocks, block_size, cache_stride, device=dev, dtype=torch.uint8
)
write(ref)
# Unified slot holding three layer pages per block; carve the middle one.
n_layers = 3
layer = 1
unified = torch.zeros(
num_blocks, n_layers, block_size, cache_stride, device=dev, dtype=torch.uint8
)
view = unified[:, layer]
assert not view.is_contiguous()
assert view.stride(0) == n_layers * block_size * cache_stride
write(view)
assert torch.equal(ref, view.contiguous())
# Zero bleed into the neighbour layers' segments.
assert unified[:, 0].abs().max().item() == 0
assert unified[:, 2].abs().max().item() == 0
def test_flashattn_mla_dense_decode_unified_slot_view():
"""FA3 decode (FLASH_ATTN_MLA backend) must read a unified-slot
block-major view bit-identically to a contiguous per-layer cache."""
try:
from vllm.vllm_flash_attn import flash_attn_varlen_func
except ImportError:
pytest.skip("vllm_flash_attn is not available")
from vllm.v1.attention.backends.fa_utils import flash_attn_supports_mla
if not flash_attn_supports_mla():
pytest.skip("FA3 MLA requires a Hopper device")
torch.manual_seed(0)
dev = "cuda"
dt = torch.bfloat16
kv_lora_rank = 512
rope_dim = 64
entry = kv_lora_rank + rope_dim # 576
h_q = 16
page = 64
num_blocks = 64
bs = 4
n_layers = 3
layer = 1
q_pe = torch.randn(bs, h_q, rope_dim, device=dev, dtype=dt) * 0.1
q_nope = torch.randn(bs, h_q, kv_lora_rank, device=dev, dtype=dt) * 0.1
kv_data = torch.randn(num_blocks, page, entry, device=dev, dtype=dt) * 0.1
# (A) contiguous per-layer reference.
cache_contiguous = kv_data.clone().contiguous()
# (B) unified slot: view one layer -> inflated stride(0), non-zero offset.
unified = torch.randn(num_blocks, n_layers, page, entry, device=dev, dtype=dt) * 0.1
unified[:, layer].copy_(kv_data)
cache_view = unified[:, layer]
assert not cache_view.is_contiguous()
assert cache_view.stride(0) == n_layers * page * entry
max_blk = num_blocks // bs
block_table = torch.arange(num_blocks, device=dev, dtype=torch.int32).view(
bs, max_blk
)
seq_lens = torch.full((bs,), max_blk * page, device=dev, dtype=torch.int32)
cu_seqlens_q = torch.arange(bs + 1, device=dev, dtype=torch.int32)
def run(cache):
kv_c_cache = cache[..., :kv_lora_rank]
k_pe_cache = cache[..., kv_lora_rank:]
out = flash_attn_varlen_func(
q=q_pe,
k=k_pe_cache.unsqueeze(-2), # Add head dim of 1
v=kv_c_cache.unsqueeze(-2), # Add head dim of 1
q_v=q_nope,
max_seqlen_q=1,
cu_seqlens_q=cu_seqlens_q,
max_seqlen_k=int(seq_lens.max().item()),
seqused_k=seq_lens,
block_table=block_table,
softmax_scale=entry**-0.5,
causal=True,
fa_version=3,
)
return out.clone().float()
out_ref = run(cache_contiguous)
out_view = run(cache_view)
assert torch.isfinite(out_ref).all()
assert out_ref.abs().max().item() > 0.0
assert (out_ref - out_view).abs().max().item() == 0.0
def test_flashmla_dense_fp8_decode_unified_slot_view():
"""FlashMLA dense fp8 decode (FLASHMLA backend with quantized KV cache)
must read a unified-slot block-major view bit-identically to a contiguous
per-layer fp8 cache."""
import vllm.v1.attention.ops.flashmla as fm
ok, reason = fm.is_flashmla_dense_supported()
if not ok:
pytest.skip(reason)
torch.manual_seed(0)
dev = "cuda"
head_dim = 576
hdv = 512
h_q = 128
page = 64
num_blocks = 64
bs = 4
n_layers = 3
layer = 1
q = torch.randn(bs, 1, h_q, head_dim, device=dev, dtype=torch.bfloat16) * 0.1
kv_data = (torch.randn(num_blocks, page, head_dim, device=dev) * 0.1).to(
torch.float8_e4m3fn
)
# (A) contiguous per-layer reference.
cache_contiguous = kv_data.clone().contiguous()
# (B) unified slot: view one layer -> inflated stride(0), non-zero offset.
unified = (torch.randn(num_blocks, n_layers, page, head_dim, device=dev) * 0.1).to(
torch.float8_e4m3fn
)
unified[:, layer].copy_(kv_data)
cache_view = unified[:, layer]
assert not cache_view.is_contiguous()
assert cache_view.stride(0) == n_layers * page * head_dim
max_blk = num_blocks // bs
block_table = torch.arange(num_blocks, device=dev, dtype=torch.int32).view(
bs, max_blk
)
cache_seqlens = torch.full((bs,), max_blk * page, device=dev, dtype=torch.int32)
descale = torch.ones(1, device=dev, dtype=torch.float32)
def run(kc):
tile_md, num_splits = fm.get_mla_metadata_dense_fp8(cache_seqlens, h_q, 1)
out, _ = fm.flash_mla_with_kvcache_fp8(
q=q,
k_cache=kc.unsqueeze(-2), # Add head dim of 1
block_table=block_table,
cache_seqlens=cache_seqlens,
head_dim_v=hdv,
tile_scheduler_metadata=tile_md,
num_splits=num_splits,
softmax_scale=head_dim**-0.5,
causal=True,
descale_q=descale,
descale_k=descale,
)
return out.clone().float()
out_ref = run(cache_contiguous)
out_view = run(cache_view)
assert torch.isfinite(out_ref).all()
assert out_ref.abs().max().item() > 0.0
assert (out_ref - out_view).abs().max().item() == 0.0
def test_flashinfer_mla_dense_fp8_decode_unified_slot_view():
"""FlashInfer MLA dense decode with an fp8 KV cache must read a
unified-slot block-major view bit-identically to a contiguous per-layer
cache."""
try:
from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla
except ImportError:
pytest.skip("flashinfer is not available")
from vllm.platforms import current_platform
if not current_platform.is_device_capability_family(100):
pytest.skip("FlashInfer trtllm-gen MLA requires sm100")
torch.manual_seed(0)
dev = "cuda"
kv_lora_rank = 512
qk_rope_head_dim = 64
qk_nope_head_dim = 128
head_dim = kv_lora_rank + qk_rope_head_dim # 576
num_qo_heads = 128
page = 64
num_blocks = 64
bs = 4
n_layers = 3
layer = 1
# With a quantized KV cache the decode query is quantized to fp8 as well
# (trtllm-gen has no bf16-query x fp8-cache decode kernel).
q = (torch.randn(bs, 1, num_qo_heads, head_dim, device=dev) * 0.1).to(
torch.float8_e4m3fn
)
kv_data = (torch.randn(num_blocks, 1, page, head_dim, device=dev) * 0.1).to(
torch.float8_e4m3fn
)
# (A) contiguous per-layer reference.
kv_contiguous = kv_data.clone().contiguous()
# (B) unified slot: view one layer -> inflated stride(0), non-zero offset.
unified = (
torch.randn(num_blocks, n_layers, 1, page, head_dim, device=dev) * 0.1
).to(torch.float8_e4m3fn)
unified[:, layer].copy_(kv_data)
kv_view = unified[:, layer]
assert not kv_view.is_contiguous()
assert kv_view.stride(0) == n_layers * 1 * page * head_dim
max_blk = num_blocks // bs
block_tables = torch.arange(num_blocks, device=dev, dtype=torch.int32).view(
bs, max_blk
)
seq_lens = torch.full((bs,), max_blk * page, device=dev, dtype=torch.int32)
ws = torch.empty(128 * 1024 * 1024, dtype=torch.int8, device=dev)
scale = head_dim**-0.5
def run(kv):
return trtllm_batch_decode_with_kv_cache_mla(
query=q,
kv_cache=kv,
workspace_buffer=ws,
qk_nope_head_dim=qk_nope_head_dim,
kv_lora_rank=kv_lora_rank,
qk_rope_head_dim=qk_rope_head_dim,
block_tables=block_tables,
seq_lens=seq_lens,
max_seq_len=int(seq_lens.max().item()),
bmm1_scale=scale,
bmm2_scale=1.0,
).clone()
out_ref = run(kv_contiguous).float()
out_view = run(kv_view).float()
assert torch.isfinite(out_ref).all()
assert (out_ref - out_view).abs().max().item() == 0.0
@@ -231,3 +231,95 @@ def test_decode_attention_fp8(B, L, H_Q, H_KV, D_QK, D_V, CACHE_SIZE, PAGE_SIZE)
# FP8 tolerances match test_mla_backends.py test_backend_correctness.
torch.testing.assert_close(o_ref, o_fp8, atol=5e-1, rtol=1e-2)
@pytest.mark.parametrize(
"H_Q,H_KV,D_QK,D_V,is_mla",
[
(16, 1, 576, 512, True), # MLA path (grouped kernel, v = trans(k))
(32, 8, 128, 128, False), # GQA path (grouped kernel)
(32, 32, 128, 128, False), # MHA path (normal kernel)
],
)
@pytest.mark.parametrize("PAGE_SIZE", [16])
def test_decode_attention_cross_layer_view(H_Q, H_KV, D_QK, D_V, is_mla, PAGE_SIZE):
"""The kernel must honor the cache's page-dim stride, not assume pages are
packed back-to-back. A per-layer view into a cross-layer (block-major)
cache has stride(0) inflated by num_layers; outputs must match a
contiguous cache holding the same data exactly."""
B = 3
seq_len = 1027
CACHE_SIZE = 16384
NUM_LAYERS = 3
LAYER_IDX = 1
dtype = torch.bfloat16
sm_scale = 1.0 / (D_QK**0.5)
num_kv_splits = 8
num_pages = CACHE_SIZE // PAGE_SIZE
num_pages_per_batch = cdiv(seq_len, PAGE_SIZE)
req_to_page = torch.randint(
0, num_pages, (B, num_pages_per_batch), device=DEVICE_TYPE
)
q = torch.randn(B, H_Q, D_QK, dtype=dtype, device=DEVICE_TYPE)
b_seq_len = torch.full((B,), seq_len, device=DEVICE_TYPE)
# Reference: contiguous paged cache.
k_ref = torch.randn(
num_pages, PAGE_SIZE, H_KV, D_QK, dtype=dtype, device=DEVICE_TYPE
)
if is_mla:
v_ref = k_ref[..., :D_V]
else:
v_ref = torch.randn(
num_pages, PAGE_SIZE, H_KV, D_V, dtype=dtype, device=DEVICE_TYPE
)
# Cross-layer cache: all layers' pages for a block are adjacent. The
# per-layer view has the same shape as the contiguous cache but
# stride(0) is NUM_LAYERS x larger. Neighbor layers hold random data so
# any packed-pages addressing reads garbage rather than zeros.
k_xl = torch.randn(
num_pages, NUM_LAYERS, PAGE_SIZE, H_KV, D_QK, dtype=dtype, device=DEVICE_TYPE
)
k_view = k_xl[:, LAYER_IDX]
k_view.copy_(k_ref)
assert k_view.stride(0) == NUM_LAYERS * PAGE_SIZE * H_KV * D_QK
if is_mla:
v_view = k_view[..., :D_V]
else:
v_xl = torch.randn(
num_pages, NUM_LAYERS, PAGE_SIZE, H_KV, D_V, dtype=dtype, device=DEVICE_TYPE
)
v_view = v_xl[:, LAYER_IDX]
v_view.copy_(v_ref)
def run(k_buffer, v_buffer):
o = torch.zeros(B, H_Q, D_V, dtype=dtype, device=DEVICE_TYPE)
lse = torch.zeros(B, H_Q, dtype=dtype, device=DEVICE_TYPE)
attn_logits = torch.empty(
(B, H_Q, num_kv_splits, D_V + 1), dtype=torch.float32, device=DEVICE_TYPE
)
decode_attention_fwd(
q,
k_buffer,
v_buffer,
o,
lse,
req_to_page,
b_seq_len,
attn_logits,
num_kv_splits,
sm_scale,
PAGE_SIZE,
is_mla=is_mla,
)
return o, lse
o_ref, lse_ref = run(k_ref, v_ref)
o_xl, lse_xl = run(k_view, v_view)
# Same data and same compute order; only addressing differs.
assert torch.equal(o_ref, o_xl)
assert torch.equal(lse_ref, lse_xl)
@@ -1,10 +1,13 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
def test_mla_backend_rejects_cross_layer_kv_cache():
"""MLA backends return identity permutation (layers dim first)
to signal cross-layer KV cache is unsupported."""
def test_mla_common_backend_rejects_cross_layer_kv_cache():
"""MLACommonBackend defaults to the identity permutation (layers dim
first) so MLA backends whose decode kernels are not verified to honor
the cache's block-dim stride stay opted out of cross-layer KV cache."""
from vllm.model_executor.layers.attention.mla_attention import (
MLACommonBackend,
)
@@ -19,6 +22,35 @@ def test_mla_backend_rejects_cross_layer_kv_cache():
) == (0, 1, 2)
@pytest.mark.parametrize(
"backend_path",
[
"vllm.v1.attention.backends.mla.triton_mla.TritonMLABackend",
"vllm.v1.attention.backends.mla.cutlass_mla.CutlassMLABackend",
"vllm.v1.attention.backends.mla.flashattn_mla.FlashAttnMLABackend",
"vllm.v1.attention.backends.mla.flashmla.FlashMLABackend",
"vllm.v1.attention.backends.mla.flashinfer_mla.FlashInferMLABackend",
],
)
def test_verified_mla_backends_support_cross_layer_kv_cache(backend_path):
"""Backends whose decode kernels honor the cache's block-dim stride opt
in to the cross-layer layout with a non-identity permutation placing
num_blocks first in physical layout."""
module_path, name = backend_path.rsplit(".", 1)
backend = getattr(
pytest.importorskip(module_path, reason="backend deps unavailable"), name
)
stride_order = backend.get_kv_cache_stride_order(include_num_layers_dimension=True)
assert stride_order == (1, 0, 2, 3)
assert stride_order[0] != 0 # num_blocks first => cross-layer supported
assert backend.get_kv_cache_stride_order(include_num_layers_dimension=False) == (
0,
1,
2,
)
def test_deepseek_v32_indexer_rejects_cross_layer_kv_cache():
"""DeepseekV32Indexer returns identity permutation (layers dim first)
to signal cross-layer KV cache is unsupported."""
@@ -1212,9 +1212,9 @@ class MLACommonBackend(AttentionBackend):
include_num_layers_dimension: bool = False,
) -> tuple[int, ...]:
if include_num_layers_dimension:
# MLA kernels require contiguous per-layer KV cache views.
# Identity permutation keeps num_layers first in physical
# layout, signaling cross-layer allocation is unsupported.
# Default to identity permutation to signal cross-layer allocation
# is unsupported. Each MLA backend must opt in to support cross-layer
# allocation by overriding this method.
return (0, 1, 2, 3)
return (0, 1, 2)
@@ -49,6 +49,14 @@ class CutlassMLABackend(MLACommonBackend):
def get_supported_kernel_block_sizes() -> list[int | MultipleOf]:
return [128]
@staticmethod
def get_kv_cache_stride_order(
include_num_layers_dimension: bool = False,
) -> tuple[int, ...]:
if include_num_layers_dimension:
return (1, 0, 2, 3)
return (0, 1, 2)
@staticmethod
def get_name() -> str:
return "CUTLASS_MLA"
@@ -52,6 +52,14 @@ class FlashAttnMLABackend(MLACommonBackend):
def get_supported_kernel_block_sizes() -> list[int | MultipleOf]:
return [MultipleOf(16)]
@staticmethod
def get_kv_cache_stride_order(
include_num_layers_dimension: bool = False,
) -> tuple[int, ...]:
if include_num_layers_dimension:
return (1, 0, 2, 3)
return (0, 1, 2)
@staticmethod
def get_name() -> str:
return "FLASH_ATTN_MLA"
@@ -49,6 +49,14 @@ class FlashInferMLABackend(MLACommonBackend):
def get_supported_kernel_block_sizes() -> list[int | MultipleOf]:
return [32, 64]
@staticmethod
def get_kv_cache_stride_order(
include_num_layers_dimension: bool = False,
) -> tuple[int, ...]:
if include_num_layers_dimension:
return (1, 0, 2, 3)
return (0, 1, 2)
@staticmethod
def get_name() -> str:
return "FLASHINFER_MLA"
@@ -58,6 +58,14 @@ class FlashMLABackend(MLACommonBackend):
def get_supported_kernel_block_sizes() -> list[int | MultipleOf]:
return [64]
@staticmethod
def get_kv_cache_stride_order(
include_num_layers_dimension: bool = False,
) -> tuple[int, ...]:
if include_num_layers_dimension:
return (1, 0, 2, 3)
return (0, 1, 2)
@staticmethod
def get_name() -> str:
return "FLASHMLA"
@@ -57,6 +57,14 @@ class TritonMLABackend(MLACommonBackend):
return True
return block_size % 16 == 0
@staticmethod
def get_kv_cache_stride_order(
include_num_layers_dimension: bool = False,
) -> tuple[int, ...]:
if include_num_layers_dimension:
return (1, 0, 2, 3)
return (0, 1, 2)
@staticmethod
def get_name() -> str:
return "TRITON_MLA"
@@ -56,6 +56,15 @@ def tanh(x):
return 2 * tl.sigmoid(2 * x) - 1
def _page_stride(buf, page_size):
# Stride between pages. 4D buffers have a page dim; 3D buffers pack pages
# along the token dim, so split it out first. Read the real stride (a
# cross-layer view has gaps), don't assume PAGE_SIZE * token stride.
if buf.ndim == 3:
buf = buf.unflatten(-3, (-1, page_size))
return buf.stride(-4)
@triton.jit
def _fwd_kernel_stage1(
Q,
@@ -68,8 +77,10 @@ def _fwd_kernel_stage1(
stride_req_to_tokens_b,
stride_qbs,
stride_qh,
stride_buf_kpbs,
stride_buf_kbs,
stride_buf_kh,
stride_buf_vpbs,
stride_buf_vbs,
stride_buf_vh,
stride_mid_ob,
@@ -123,9 +134,11 @@ def _fwd_kernel_stage1(
mask=offs_n < split_kv_end,
other=0,
)
kv_loc = kv_page_number * PAGE_SIZE + offs_n % PAGE_SIZE
kv_in_page = offs_n % PAGE_SIZE
offs_buf_k = (
kv_loc[:, None] * stride_buf_kbs
(kv_page_number * stride_buf_kpbs + kv_in_page * stride_buf_kbs)[
:, None
]
+ cur_kv_head * stride_buf_kh
+ offs_d[None, :]
)
@@ -145,7 +158,9 @@ def _fwd_kernel_stage1(
qk = tl.where(offs_n < split_kv_end, qk, float("-inf"))
offs_buf_v = (
kv_loc[:, None] * stride_buf_vbs
(kv_page_number * stride_buf_vpbs + kv_in_page * stride_buf_vbs)[
:, None
]
+ cur_kv_head * stride_buf_vh
+ offs_dv[None, :]
)
@@ -235,8 +250,10 @@ def _decode_att_m_fwd(
Req_to_tokens.stride(0),
q.stride(0),
q.stride(1),
_page_stride(k_buffer, page_size),
k_buffer.stride(-3), # Assume (..., PAGE_SIZE, NUM_HEADS, HEAD_DIM)
k_buffer.stride(-2), # Assume (..., PAGE_SIZE, NUM_HEADS, HEAD_DIM)
_page_stride(v_buffer, page_size),
v_buffer.stride(-3), # Assume (..., PAGE_SIZE, NUM_HEADS, HEAD_DIM)
v_buffer.stride(-2), # Assume (..., PAGE_SIZE, NUM_HEADS, HEAD_DIM)
att_out.stride(0),
@@ -270,8 +287,10 @@ def _fwd_grouped_kernel_stage1(
stride_req_to_tokens_b,
stride_qbs,
stride_qh,
stride_buf_kpbs,
stride_buf_kbs,
stride_buf_kh,
stride_buf_vpbs,
stride_buf_vbs,
stride_buf_vh,
stride_mid_ob,
@@ -357,10 +376,12 @@ def _fwd_grouped_kernel_stage1(
other=0,
cache_modifier=".ca",
)
kv_loc = kv_page_number * PAGE_SIZE + offs_n % PAGE_SIZE
kv_off_k = (
kv_page_number * stride_buf_kpbs + (offs_n % PAGE_SIZE) * stride_buf_kbs
)
# explicitly facilitate overlapping load/compute
offs_buf_k = kv_loc[None, :] * stride_buf_kbs + base_offs_k
offs_buf_k = kv_off_k[None, :] + base_offs_k
k = tl.load(
K_Buffer + offs_buf_k,
mask=(offs_n[None, :] < split_kv_end) & (mask_d[:, None]),
@@ -372,7 +393,7 @@ def _fwd_grouped_kernel_stage1(
k = (k.to(tl.float32) * ks).to(q.dtype)
qk = tl.dot(q, k.to(q.dtype))
if BLOCK_DPE > 0:
offs_buf_kpe = kv_loc[None, :] * stride_buf_kbs + base_offs_kpe
offs_buf_kpe = kv_off_k[None, :] + base_offs_kpe
kpe = tl.load(
K_Buffer + offs_buf_kpe,
mask=(offs_n[None, :] < split_kv_end) & (mask_dpe[:, None]),
@@ -392,7 +413,11 @@ def _fwd_grouped_kernel_stage1(
)
if not IS_MLA:
offs_buf_v = kv_loc[:, None] * stride_buf_vbs + base_offs_v
kv_off_v = (
kv_page_number * stride_buf_vpbs
+ (offs_n % PAGE_SIZE) * stride_buf_vbs
)
offs_buf_v = kv_off_v[:, None] + base_offs_v
v = tl.load(
V_Buffer + offs_buf_v,
mask=(offs_n[:, None] < split_kv_end) & (mask_dv[None, :]),
@@ -517,8 +542,10 @@ def _decode_grouped_att_m_fwd(
Req_to_tokens.stride(0),
q.stride(0),
q.stride(1),
_page_stride(k_buffer, page_size),
k_buffer.stride(-3), # Assume (..., PAGE_SIZE, NUM_HEADS, HEAD_DIM)
k_buffer.stride(-2), # Assume (..., PAGE_SIZE, NUM_HEADS, HEAD_DIM)
_page_stride(v_buffer, page_size),
v_buffer.stride(-3), # Assume (..., PAGE_SIZE, NUM_HEADS, HEAD_DIM)
v_buffer.stride(-2), # Assume (..., PAGE_SIZE, NUM_HEADS, HEAD_DIM)
att_out.stride(0),