[2/N][KV-Cache Layout Refactor] Pack K/V into the content dim across attention backends (#44455)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
Lucas Wilkinson
2026-07-11 11:11:16 -04:00
committed by GitHub
co-authored by Claude OpenAI Codex Matthew Bonanni Nicolò Lucchesi
parent 76fedaa2a5
commit 51878e5b6e
31 changed files with 664 additions and 604 deletions
@@ -288,16 +288,15 @@ __global__ void fusedMiniMaxM3QNormRopeKVInsertKernel(
int64_t const* __restrict__ positions, // [N] i64
int64_t const* __restrict__ slot_mapping, // main K/V slots or nullptr
int64_t const* __restrict__ index_slot_mapping, // index K slots/nullptr
cache_t* __restrict__ kv_cache, // [nb,2,bs,nkv,128] or nullptr
cache_t* __restrict__ kv_cache, // [nb,nkv,bs,2*128] or nullptr
out_idx_t* __restrict__ index_cache, // [nb*bs, 128]; scalar_t or e4m3 byte
float const eps, int const rotary_dim, int const num_tokens, int const nq,
int const nkv, int const niq, int const block_size,
// kv_cache strides (in elements) for logical shape [nb, 2, bs, nkv, 128].
// The head_dim (last) dim is always innermost-contiguous (stride 1), so the
// NHD/HND layout choice is fully captured by these four strides: NHD keeps
// s_token < s_head, HND swaps them. dim_base addresses head_dim directly.
int64_t const kv_s_block, int64_t const kv_s_kv, int64_t const kv_s_token,
int64_t const kv_s_head) {
// kv_cache strides (in elements) for logical shape [nb, nkv, bs, 2*128].
// The content (last) dim is always innermost-contiguous (stride 1), so the
// NHD/HND layout choice is captured by the head/token strides.
int64_t const kv_s_block, int64_t const kv_s_head, int64_t const kv_s_token,
int64_t const kv_s_dim) {
#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)
// _typeConvert<BFloat16> is unavailable on pre-Ampere; the M3 kernel only
// runs with bf16/fp16 inputs in practice. Discard the bf16 body there.
@@ -438,16 +437,16 @@ __global__ void fusedMiniMaxM3QNormRopeKVInsertKernel(
storeElems<scalar_t>(index_cache + sm * kHeadDim + dim_base, elems);
}
} else if (isK || isV) {
// kv_cache logical shape [num_blocks, 2, block_size, nkv, head_dim].
// kv_cache logical shape [num_blocks, nkv, block_size, 2*head_dim].
// Paging is logical (block = sm/block_size, token = sm%block_size);
// the physical NHD/HND layout is honoured via the passed strides.
int64_t const b = sm / block_size;
int64_t const t = sm % block_size;
int const kv = isK ? 0 : 1;
int64_t const off =
b * kv_s_block + kv * kv_s_kv + t * kv_s_token + head * kv_s_head;
storeCacheElems<scalar_t, cache_t, kv_dt>(kv_cache + off + dim_base,
elems);
int64_t const off = b * kv_s_block + head * kv_s_head +
t * kv_s_token +
(kv * kHeadDim + dim_base) * kv_s_dim;
storeCacheElems<scalar_t, cache_t, kv_dt>(kv_cache + off, elems);
}
}
}
@@ -474,8 +473,8 @@ void launchFusedMiniMaxM3(
int64_t const* index_slot_mapping, cache_t* kv_cache, void* index_cache,
float const eps, int const rotary_dim, int const num_tokens, int const nq,
int const nkv, int const niq, int const block_size,
int64_t const kv_s_block, int64_t const kv_s_kv, int64_t const kv_s_token,
int64_t const kv_s_head, bool const has_index, bool const insert_kv,
int64_t const kv_s_block, int64_t const kv_s_head, int64_t const kv_s_token,
int64_t const kv_s_dim, bool const has_index, bool const insert_kv,
bool const fp8_idx, cudaStream_t stream) {
// Index outputs are scalar_t (bf16) or e4m3 bytes (uint8_t); reinterpret the
// void* pointers per instantiation in the LAUNCH macro.
@@ -517,20 +516,20 @@ void launchFusedMiniMaxM3(
iq_norm_w, ik_norm_w, cos_sin_cache, positions, slot_mapping, \
index_slot_mapping, kv_cache, reinterpret_cast<OUT_T*>(index_cache), \
eps, rotary_dim, num_tokens, nq, nkv, niq, block_size, kv_s_block, \
kv_s_kv, kv_s_token, kv_s_head)
kv_s_head, kv_s_token, kv_s_dim)
#else
// ROCm: standard kernel launch syntax (no PDL/stream serialization).
// clang-format off
#define LAUNCH(IS_SPARSE, INSERT, FP8, OUT_T) \
fusedMiniMaxM3QNormRopeKVInsertKernel<scalar_t, cache_t, kv_dt, OUT_T, \
fusedMiniMaxM3QNormRopeKVInsertKernel<scalar_t, cache_t, kv_dt, OUT_T, \
IS_SPARSE, INSERT, FP8> \
<<<grid, kBlockSize, 0, stream>>>( \
qkv, q_out, reinterpret_cast<OUT_T*>(index_q_out), q_norm_w, \
k_norm_w, iq_norm_w, ik_norm_w, cos_sin_cache, positions, \
slot_mapping, index_slot_mapping, kv_cache, \
reinterpret_cast<OUT_T*>(index_cache), eps, rotary_dim, \
num_tokens, nq, nkv, niq, block_size, kv_s_block, kv_s_kv, \
kv_s_token, kv_s_head)
num_tokens, nq, nkv, niq, block_size, kv_s_block, kv_s_head, \
kv_s_token, kv_s_dim)
// clang-format on
#endif
@@ -584,8 +583,8 @@ void launchFusedMiniMaxM3(
? reinterpret_cast<void*>(index_cache->data_ptr()) \
: nullptr, \
static_cast<float>(eps), static_cast<int>(rotary_dim), num_tokens, nq, \
nkv, niq, static_cast<int>(block_size), kv_s_block, kv_s_kv, kv_s_token, \
kv_s_head, has_index, insert_kv, fp8_idx, stream)
nkv, niq, static_cast<int>(block_size), kv_s_block, kv_s_head, \
kv_s_token, kv_s_dim, has_index, insert_kv, fp8_idx, stream)
// ────────────────────────────────────────────────────────────────────────────
// Torch op wrapper
@@ -602,7 +601,7 @@ void fused_minimax_m3_qknorm_rope_kv_insert(
int64_t num_index_heads, // niq; 0 => dense
std::optional<torch::stable::Tensor> slot_mapping, // [N] i64
std::optional<torch::stable::Tensor> index_slot_mapping, // [N] i64
std::optional<torch::stable::Tensor> kv_cache, // [nb,2,bs,nkv,128]
std::optional<torch::stable::Tensor> kv_cache, // [nb,nkv,bs,2*128]
std::optional<torch::stable::Tensor> index_cache, // [nb,bs,128]
int64_t block_size,
std::optional<torch::stable::Tensor> q_out, // [N, nq*128] contiguous
@@ -673,11 +672,11 @@ void fused_minimax_m3_qknorm_rope_kv_insert(
index_k_norm_weight->numel() == kHeadDim,
"index norm weights must have 128 elements");
}
// kv_cache strides (logical shape [nb, 2, bs, nkv, head_dim]). Read straight
// kv_cache strides (logical shape [nb, nkv, bs, 2*head_dim]). Read straight
// off the tensor so the kernel honours whatever physical layout the attention
// backend allocated (NHD: stride order (0,1,2,3,4); HND: (0,1,3,2,4)). No new
// backend allocated (NHD: stride order (0,2,1,3); HND: (0,1,2,3)). No new
// op argument is needed -- the strides ride along with the tensor itself.
int64_t kv_s_block = 0, kv_s_kv = 0, kv_s_token = 0, kv_s_head = 0;
int64_t kv_s_block = 0, kv_s_head = 0, kv_s_token = 0, kv_s_dim = 0;
torch::stable::Tensor const* effective_index_slot_mapping = nullptr;
if (insert_kv) {
STD_TORCH_CHECK(
@@ -707,13 +706,13 @@ void fused_minimax_m3_qknorm_rope_kv_insert(
index_cache->scalar_type() ==
torch::headeronly::ScalarType::Float8_e4m3fn),
"insert mode requires index_cache matching qkv dtype or fp8 e4m3");
STD_TORCH_CHECK(kv_cache->dim() == 5 && kv_cache->stride(4) == 1,
"kv_cache must be [nb,2,bs,nkv,head_dim] with contiguous "
"head_dim (stride(4)==1)");
STD_TORCH_CHECK(kv_cache->dim() == 4 && kv_cache->stride(3) == 1,
"kv_cache must be [nb,nkv,bs,2*head_dim] with contiguous "
"content dim (stride(3)==1)");
kv_s_block = kv_cache->stride(0);
kv_s_kv = kv_cache->stride(1);
kv_s_head = kv_cache->stride(1);
kv_s_token = kv_cache->stride(2);
kv_s_head = kv_cache->stride(3);
kv_s_dim = kv_cache->stride(3);
effective_index_slot_mapping = index_slot_mapping.has_value()
? &index_slot_mapping.value()
: &slot_mapping.value();
+6 -3
View File
@@ -111,7 +111,11 @@ class AttentionQuantPatternModel(torch.nn.Module):
# Fetch the attention backend and kv cache shape and stride order
attn_backend = self.attn.attn_backend
kv_cache_shape = attn_backend.get_kv_cache_shape(
num_blocks, self.block_size, self.num_kv_heads, self.head_size
num_blocks,
self.block_size,
self.num_kv_heads,
self.head_size,
cache_dtype_str=self.attn.kv_cache_dtype,
)
try:
kv_cache_stride_order = attn_backend.get_kv_cache_stride_order()
@@ -125,11 +129,10 @@ class AttentionQuantPatternModel(torch.nn.Module):
# Create dummy KV cache
raw_tensor = torch.zeros(
2 * num_blocks * self.block_size * self.num_kv_heads * self.head_size,
kv_cache_shape,
dtype=self.attn.kv_cache_torch_dtype,
device=self.device,
)
raw_tensor = raw_tensor.view(kv_cache_shape)
kv_cache = raw_tensor.permute(*inv_order)
self.attn.kv_cache = kv_cache
+3 -5
View File
@@ -10,7 +10,7 @@ from tests.kernels.utils import DEFAULT_OPCHECK_TEST_UTILS, opcheck
from vllm import _custom_ops as ops
from vllm.model_executor.layers.quantization.utils.quant_utils import scaled_dequantize
from vllm.platforms import current_platform
from vllm.utils.torch_utils import nvfp4_kv_cache_split_views, set_random_seed
from vllm.utils.torch_utils import nvfp4_split_data_scale, set_random_seed
COPYING_DIRECTION = [("cuda", "cpu"), ("cuda", "cuda"), ("cpu", "cuda")]
DTYPES = [torch.bfloat16, torch.float]
@@ -255,10 +255,8 @@ def test_reshape_and_cache_flash(
nvfp4_key_data = None
nvfp4_value_data = None
if kv_cache_dtype == "nvfp4":
(nvfp4_key_data,), (key_scale_cache,) = nvfp4_kv_cache_split_views(key_cache)
(nvfp4_value_data,), (value_scale_cache,) = nvfp4_kv_cache_split_views(
value_cache
)
nvfp4_key_data, key_scale_cache = nvfp4_split_data_scale(key_cache)
nvfp4_value_data, value_scale_cache = nvfp4_split_data_scale(value_cache)
if kv_cache_dtype == "nvfp4":
# Global scale = amax / 448 (per-tensor)
@@ -13,7 +13,7 @@ from vllm.platforms import current_platform
from vllm.utils.math_utils import round_up
from vllm.utils.torch_utils import (
nvfp4_kv_cache_full_dim,
nvfp4_kv_cache_split_views,
nvfp4_split_data_scale,
set_random_seed,
)
@@ -74,17 +74,18 @@ def make_nvfp4_kv_cache(
kv_scale_val, dtype=torch.float32, device=kv_bf16_hnd.device
)
# Allocate in HND physical order, permute to NHD logical order.
# hnd_order swaps dims 2↔3; it is its own inverse.
# layout: (B, 2*H, N, full_dim)
# where K heads occupy the first H heads and V heads occupy the second H heads.
full_dim = nvfp4_kv_cache_full_dim(head_size)
hnd_order = (0, 1, 3, 2, 4)
kv_cache = torch.zeros(
(num_blocks, 2, num_kv_heads, block_size, full_dim),
kv_cache_hnd = torch.zeros(
(num_blocks, 2 * num_kv_heads, block_size, full_dim),
dtype=torch.uint8,
device=kv_bf16_hnd.device,
).permute(*hnd_order)
)
kv_cache_nhd = kv_cache_hnd.permute(0, 2, 1, 3)
k_view_nhd, v_view_nhd = kv_cache_nhd.split(num_kv_heads, dim=-2)
# Flatten NHD [N, T, H, D] → token tensors [N*T, H, D] for the kernel.
# Flatten input KV → token tensors [B*N, H, head_size] for the kernel.
num_tokens = num_blocks * block_size
k_tokens = (
kv_bf16_hnd[:, 0]
@@ -98,22 +99,21 @@ def make_nvfp4_kv_cache(
)
slot_mapping = torch.arange(num_tokens, dtype=torch.long, device=kv_bf16_hnd.device)
# reshape_and_cache_flash: kernel receives kv_cache[:, 0] and [:, 1]
# (full K/V buffers containing both data and scale).
torch.ops._C_cache_ops.reshape_and_cache_flash(
k_tokens,
v_tokens,
kv_cache[:, 0],
kv_cache[:, 1],
k_view_nhd,
v_view_nhd,
slot_mapping,
"nvfp4",
kv_scale_tensor,
kv_scale_tensor,
)
# Split in HND order for trtllm kernel (expects HND numTokensPerPage).
kv_cache_hnd = kv_cache.permute(*hnd_order)
(k_data, v_data), (k_scales, v_scales) = nvfp4_kv_cache_split_views(kv_cache_hnd)
# Split into data/scale views in HNC order for trtllm kernel.
k_cache_hnc, v_cache_hnc = kv_cache_hnd.split(num_kv_heads, dim=1)
k_data, k_scales = nvfp4_split_data_scale(k_cache_hnc)
v_data, v_scales = nvfp4_split_data_scale(v_cache_hnc)
# Dequantize for the FA2 reference baseline.
ref_k = dequant_nvfp4_kv_cache(
+40 -37
View File
@@ -662,8 +662,9 @@ def _reference_sparse_attn(
positions = torch.arange(seq_len, device="cuda")
pages = block_table[req_id, positions // BLOCK_SIZE]
rows = positions % BLOCK_SIZE
k_req = kv_cache[pages, 0, rows]
v_req = kv_cache[pages, 1, rows].float()
kv_req = kv_cache[pages, :, rows]
k_req = kv_req[..., :HEAD_DIM]
v_req = kv_req[..., HEAD_DIM:].float()
q_pos = prefix_len + torch.arange(q_len, device="cuda")
key_blocks = positions // BLOCK_SIZE
@@ -791,15 +792,15 @@ def test_main_backend_layout_contract():
flash_attn-style stride order for each layout."""
nb, bs, h, d = 7, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
logical = MiniMaxM3SparseBackend.get_kv_cache_shape(nb, bs, h, d)
assert logical == (nb, 2, bs, h, d)
# The old HND-ordered shape is no longer the logical shape.
assert logical != (nb, 2, h, bs, d)
assert logical == (nb, h, bs, 2 * d)
# The old separate K/V-axis shape is no longer the logical shape.
assert logical != (nb, 2, bs, h, d)
try:
set_kv_cache_layout("HND")
assert MiniMaxM3SparseBackend.get_kv_cache_stride_order() == (0, 1, 3, 2, 4)
assert MiniMaxM3SparseBackend.get_kv_cache_stride_order() == (0, 1, 2, 3)
set_kv_cache_layout("NHD")
assert MiniMaxM3SparseBackend.get_kv_cache_stride_order() == (0, 1, 2, 3, 4)
assert MiniMaxM3SparseBackend.get_kv_cache_stride_order() == (0, 2, 1, 3)
finally:
set_kv_cache_layout(None)
@@ -829,7 +830,7 @@ def test_main_backend_unknown_layout_raises(monkeypatch):
def test_indexer_backend_stride_order_is_identity():
"""The 3-dim indexer cache must not inherit the parent's 5-element stride
"""The 3-dim indexer cache must not inherit the parent's 4-element stride
order; it overrides to the 3-element identity so the allocator keeps the
contiguous layout."""
assert MiniMaxM3IndexerBackend.get_kv_cache_stride_order() == (0, 1, 2)
@@ -848,9 +849,9 @@ def test_indexer_backend_stride_order_is_identity():
assert _stride_order_for(MiniMaxM3IndexerBackend, len(indexer_shape)) == (0, 1, 2)
def test_hnd_allocation_is_byte_identical_to_transpose():
"""Under HND the backend-visible logical view is byte-identical to the
pre-change allocate-HND-then-transpose(2, 3) workaround."""
def test_hnd_allocation_is_packed_head_major():
"""Under HND the backend-visible logical view is the packed head-major
physical allocation."""
nb, bs, h, d = 4, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
logical = MiniMaxM3SparseBackend.get_kv_cache_shape(nb, bs, h, d)
try:
@@ -860,21 +861,22 @@ def test_hnd_allocation_is_byte_identical_to_transpose():
set_kv_cache_layout(None)
physical_shape = tuple(logical[i] for i in stride_order)
# The physical (permuted) shape equals the old hardcoded HND shape.
assert physical_shape == (nb, 2, h, bs, d)
assert physical_shape == (nb, h, bs, 2 * d)
inv_order = [stride_order.index(i) for i in range(len(stride_order))]
raw = torch.empty(physical_shape, device="cuda", dtype=DTYPE)
view = raw.permute(*inv_order)
expected = raw.view((nb, 2, h, bs, d)).transpose(2, 3)
expected = raw.view((nb, h, bs, 2 * d))
assert view.shape == expected.shape
assert view.stride() == expected.stride()
assert view.storage_offset() == expected.storage_offset()
# Negative: the identity (wrong) stride order under HND does not reproduce
# the transpose view.
wrong_view = raw.view(logical)
# Negative: the NHD stride order under HND does not reproduce the
# head-major view.
wrong_order = (0, 2, 1, 3)
wrong_inv = [wrong_order.index(i) for i in range(len(wrong_order))]
wrong_view = raw.view(tuple(logical[i] for i in wrong_order)).permute(*wrong_inv)
assert wrong_view.stride() != expected.stride()
@@ -1037,14 +1039,18 @@ def test_decode_wrong_layout_breaks_parity():
seq_lens_list = (130, 257)
q, block_table, seq_lens, topk_idx, num_pages = _build_decode_inputs(seq_lens_list)
# Physical HND storage [blocks, 2, heads, block, dim].
# Physical HND storage [blocks, heads, block, packed_kv_dim].
phys = torch.randn(
(num_pages, 2, NUM_KV_HEADS, BLOCK_SIZE, HEAD_DIM), device="cuda", dtype=DTYPE
(num_pages, NUM_KV_HEADS, BLOCK_SIZE, 2 * HEAD_DIM),
device="cuda",
dtype=DTYPE,
)
# Correct logical packed-HND view vs. the same bytes mislabeled as NHD
# physical storage and then exposed as a logical cache.
correct = phys
wrong = phys.reshape(num_pages, BLOCK_SIZE, NUM_KV_HEADS, 2 * HEAD_DIM).permute(
0, 2, 1, 3
)
# Correct logical-NHD view (strided) vs. the same bytes mislabeled as a
# contiguous-NHD cache — same shape, different content mapping.
correct = phys.permute(0, 1, 3, 2, 4)
wrong = phys.reshape(num_pages, 2, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM)
q_lens_t = torch.ones(len(seq_lens_list), device="cuda", dtype=torch.int32)
prefix_lens = seq_lens - q_lens_t
@@ -1072,9 +1078,9 @@ def _make_attn_group(backend, spec):
def test_main_cache_byte_identical_through_production_allocator():
"""AC-2: drive the real allocator (`_reshape_kv_cache`) for the M3 main
`FullAttentionSpec` under HND and assert the backend-visible view has the
same shape, stride, and storage offset as the pre-change
allocate-HND-then-transpose path; the indexer `MLAAttentionSpec` allocates
through the same path to its 3-dim shape."""
same shape, stride, and storage offset as the packed-HND allocation; the
indexer `MLAAttentionSpec` allocates through the same path to its 3-dim
shape."""
nb = 4
spec = FullAttentionSpec(
block_size=BLOCK_SIZE,
@@ -1092,8 +1098,7 @@ def test_main_cache_byte_identical_through_production_allocator():
set_kv_cache_layout(None)
view = kv_caches["main"]
oracle = raw.view(DTYPE).view((nb, 2, NUM_KV_HEADS, BLOCK_SIZE, HEAD_DIM))
oracle = oracle.transpose(2, 3)
oracle = raw.view(DTYPE).view((nb, NUM_KV_HEADS, BLOCK_SIZE, 2 * HEAD_DIM))
assert tuple(view.shape) == tuple(oracle.shape)
assert view.stride() == oracle.stride()
assert view.storage_offset() == oracle.storage_offset()
@@ -1119,13 +1124,13 @@ def test_main_cache_byte_identical_through_production_allocator():
def test_indexer_inherited_stride_order_trips_allocator_assert():
"""AC-4 negative: without the indexer override, the inherited 5-element
"""AC-4 negative: without the indexer override, the inherited 4-element
stride order trips the allocator's `len(stride_order) == len(shape)` assert
for the 3-dim indexer shape; the `AssertionError` is NOT swallowed by the
allocator's `(AttributeError, NotImplementedError)` fallback."""
class _BrokenIndexerBackend(MiniMaxM3IndexerBackend):
# Simulate inheriting the parent's 5-element stride order.
# Simulate inheriting the parent's 4-element stride order.
get_kv_cache_stride_order = staticmethod(
MiniMaxM3SparseBackend.get_kv_cache_stride_order
)
@@ -1192,10 +1197,8 @@ def test_padded_main_cache_is_flagged():
@pytest.mark.parametrize("kv_layout", ["NHD", "HND"], indirect=True)
def test_reshape_and_cache_flash_write_persists(kv_layout: str):
"""AC-5 write path: the `reshape_and_cache_flash` write site now consumes
`self.kv_cache.unbind(1)` directly. Writing through those views must persist
into the bound storage (read back through an independent logical view) under
both layouts — a `.contiguous()` copy of the unbind slice would leave the
bound storage unchanged."""
packed-content K/V split views. Writing through those views must persist
into the bound storage under both layouts."""
torch.manual_seed(0)
num_pages = 4
kv_cache = _allocate_main_kv_via_contract(num_pages)
@@ -1203,7 +1206,7 @@ def test_reshape_and_cache_flash_write_persists(kv_layout: str):
kv_cache.zero_()
# Exactly the production write-site code under test.
key_cache, value_cache = kv_cache.unbind(1)
key_cache, value_cache = kv_cache.transpose(1, 2).split(HEAD_DIM, dim=-1)
num_tokens = 12
slot_mapping = torch.randperm(num_pages * BLOCK_SIZE, device="cuda")[
@@ -1222,5 +1225,5 @@ def test_reshape_and_cache_flash_write_persists(kv_layout: str):
for t in range(num_tokens):
slot = int(slot_mapping[t].item())
blk, intra = divmod(slot, BLOCK_SIZE)
torch.testing.assert_close(kv_cache[blk, 0, intra], key[t])
torch.testing.assert_close(kv_cache[blk, 1, intra], value[t])
torch.testing.assert_close(kv_cache[blk, :, intra, :HEAD_DIM], key[t])
torch.testing.assert_close(kv_cache[blk, :, intra, HEAD_DIM:], value[t])
+9 -13
View File
@@ -140,12 +140,10 @@ def create_and_prepopulate_kv_cache(
block_table = common_attn_metadata.block_table_tensor
slot_mapping = common_attn_metadata.slot_mapping
# Create KV cache and populate in (2, num_blocks, ...) layout for easy
# flat indexing, then transpose to (num_blocks, 2, ...) layout.
kv_cache = torch.zeros(
2, num_blocks, block_size, num_kv_heads, head_size, dtype=dtype, device=device
num_blocks, block_size, num_kv_heads, 2 * head_size, dtype=dtype, device=device
)
kv_cache_flat = kv_cache.view(2, -1, num_kv_heads, head_size)
kv_cache_flat = kv_cache.view(-1, num_kv_heads, 2 * head_size)
# Populate the cache with the context tokens
# Start from block_id=1 since block_id=0 is considered the null block
@@ -154,15 +152,12 @@ def create_and_prepopulate_kv_cache(
k_context, v_context = k_contexts[i], v_contexts[i]
start = start_block_idx * block_size
end = start + k_context.shape[0]
kv_cache_flat[0, start:end, ...] = k_context
kv_cache_flat[1, start:end, ...] = v_context
kv_cache_flat[start:end, :, :head_size] = k_context
kv_cache_flat[start:end, :, head_size:] = v_context
# Stay block aligned and allocate enough blocks for the new tokens
start_block_idx += cdiv(int(seq_lens[i]), block_size)
# Transpose to (num_blocks, 2, ...) layout
kv_cache = kv_cache.transpose(0, 1).contiguous()
blocks_end = start_block_idx
# Permute the context blocks (excluding block 0 which is null)
@@ -199,7 +194,8 @@ def create_and_prepopulate_kv_cache(
i, block_indices
] * block_size + token_inter_block_offsets.to(device)
return kv_cache
# Transpose to logical (num_blocks, num_kv_heads, block_size, 2*hs)
return kv_cache.transpose(1, 2).contiguous()
class MockAttentionLayer:
@@ -496,9 +492,6 @@ def _test_backend_correctness(
set_kv_cache_layout("HND")
reset_kv_cache_layout = True
# Apply stride order like runtime does in
# _reshape_kv_cache (attn_utils.py:182-210): permute to physical
# layout, make contiguous, then permute to logical layout.
kv_cache_for_backend = kv_cache
if backend_cls is not None:
try:
@@ -506,6 +499,9 @@ def _test_backend_correctness(
except (AttributeError, NotImplementedError):
stride_order = tuple(range(kv_cache.ndim))
if stride_order != tuple(range(kv_cache.ndim)):
# Apply stride order like runtime does in
# _reshape_kv_cache (attn_utils.py:182-210): permute to physical
# layout, make contiguous, then permute to logical layout.
inv_order = [stride_order.index(i) for i in range(len(stride_order))]
kv_cache_for_backend = (
kv_cache.permute(*stride_order).contiguous().permute(*inv_order)
@@ -97,13 +97,13 @@ def _create_hnd_kv_cache(
device,
num_blocks,
common_attn_metadata,
kv_in_head_dim=False,
):
"""Create and populate a KV cache with HND-compatible strides.
"""Create and populate a packed KV cache with HND-compatible strides.
The returned tensor has logical shape
(num_blocks, 2, block_size, num_kv_heads, head_size) but is physically
laid out as (num_blocks, 2, num_kv_heads, block_size, head_size) so that
``kv_cache.permute(0, 1, 3, 2, 4)`` yields a contiguous HND view.
When kv_in_head_dim=False (default), returns (B, H, N, 2*hs) with K/V
packed in the content dim. When kv_in_head_dim=True, returns
(B, 2*H, N, hs) with K/V as separate head groups.
"""
seq_lens = common_attn_metadata.seq_lens.cpu()
query_lens = (
@@ -114,26 +114,34 @@ def _create_hnd_kv_cache(
slot_mapping = common_attn_metadata.slot_mapping
batch_size = len(k_contexts)
# Build cache in (2, num_blocks, block_size, num_kv_heads, head_size)
# then convert to HND format (same approach as test_attention_backends.py).
kv_cache_raw = torch.zeros(
2,
# kv_in_head_dim: (B, N, 2*H, hs) — K/V as separate head groups
# else: (B, N, H, 2*hs) — K/V packed in content dim
n_heads, content = (
(2 * num_kv_heads, head_size)
if kv_in_head_dim
else (num_kv_heads, 2 * head_size)
)
kv_cache = torch.zeros(
num_blocks,
block_size,
num_kv_heads,
head_size,
n_heads,
content,
dtype=dtype,
device=device,
)
kv_cache_flat = kv_cache_raw.view(2, -1, num_kv_heads, head_size)
kv_cache_flat = kv_cache.view(-1, n_heads, content)
start_block_idx = 1
for i in range(batch_size):
k_ctx, v_ctx = k_contexts[i], v_contexts[i]
start = start_block_idx * block_size
end = start + k_ctx.shape[0]
kv_cache_flat[0, start:end] = k_ctx
kv_cache_flat[1, start:end] = v_ctx
if kv_in_head_dim:
kv_cache_flat[start:end, :num_kv_heads] = k_ctx
kv_cache_flat[start:end, num_kv_heads:] = v_ctx
else:
kv_cache_flat[start:end, :, :head_size] = k_ctx
kv_cache_flat[start:end, :, head_size:] = v_ctx
start_block_idx += cdiv(int(seq_lens[i]), block_size)
blocks_end = start_block_idx
@@ -142,7 +150,7 @@ def _create_hnd_kv_cache(
perm = torch.randperm(blocks_end - 1) + 1
inv_perm = torch.zeros(blocks_end, dtype=torch.long, device=device)
inv_perm[1:] = torch.argsort(perm) + 1
kv_cache_raw[:, 1:blocks_end] = kv_cache_raw[:, perm]
kv_cache[1:blocks_end] = kv_cache[perm]
# Build block table.
start_block_idx = 1
@@ -165,10 +173,8 @@ def _create_hnd_kv_cache(
i, block_indices
] * block_size + intra_block_offsets.to(device)
# Transpose to FlashInfer logical shape then make HND-strided.
kv_cache = kv_cache_raw.transpose(0, 1)
kv_cache = kv_cache.transpose(2, 3).contiguous().transpose(2, 3)
return kv_cache
# Transpose to canonical: (B, H, N, 2*hs) or (B, 2*H, N, hs)
return kv_cache.transpose(1, 2).contiguous()
def _create_nvfp4_hnd_kv_cache(
@@ -187,20 +193,15 @@ def _create_nvfp4_hnd_kv_cache(
reshape_and_cache_flash, using the same block-table layout as
_create_hnd_kv_cache.
The returned tensor is dtype ``uint8`` with shape
``(num_blocks, 2, block_size, num_kv_heads, full_dim)`` in logical
(NHD) order, but physically permuted to HND layout via stride order
``(0, 1, 3, 2, 4)`` (i.e. ``num_kv_heads`` before ``block_size``).
The last dimension ``full_dim = head_size // 2 + head_size // 16``
packs two regions contiguously:
The returned tensor is dtype ``uint8`` with head-group layout
``(num_blocks, 2 * num_kv_heads, block_size, full_dim)``
where K heads occupy the first ``num_kv_heads`` heads and V heads the second.
Each ``full_dim = head_size // 2 + head_size // 16`` block packs two regions:
- **FP4 data** (``head_size // 2`` bytes): pairs of E2M1 values,
two per byte.
- **FP8 block scales** (``head_size // 16`` bytes): one E4M3
scale per 16-element block.
Dimension 1 indexes K (``[:, 0]``) and V (``[:, 1]``).
Args:
k_contexts: List of key context tensors, one per sequence.
v_contexts: List of value context tensors, one per sequence.
@@ -219,6 +220,7 @@ def _create_nvfp4_hnd_kv_cache(
``torch.Tensor``: The nvfp4 kv_cache tensor (uint8, HND-strided).
"""
# First create a bf16 HND cache so block tables are populated.
# Use kv_in_head_dim=True so K/V are separate head groups (B, 2*H, N, hs).
bf16_cache = _create_hnd_kv_cache(
k_contexts,
v_contexts,
@@ -229,20 +231,20 @@ def _create_nvfp4_hnd_kv_cache(
device,
num_blocks,
common_attn_metadata,
kv_in_head_dim=True,
)
# Allocate nvfp4 cache: same shape but with full_dim (data + scale).
# (num_blocks, 2 * num_kv_heads, block_size, full_dim) — K heads first, then V heads
full_dim = nvfp4_kv_cache_full_dim(head_size)
hnd_order = (0, 1, 3, 2, 4)
nvfp4_cache = torch.zeros(
(num_blocks, 2, num_kv_heads, block_size, full_dim),
(num_blocks, 2 * num_kv_heads, block_size, full_dim),
dtype=torch.uint8,
device=device,
).permute(*hnd_order)
)
k_cache, v_cache = nvfp4_cache.split(num_kv_heads, dim=1)
# Flatten bf16 context into tokens and quantize via reshape_and_cache_flash.
# bf16_cache is (num_blocks, 2, block_size, num_kv_heads, head_size) logical
# with HND physical strides.
# bf16_cache is (B, 2*H, N, hs); split K/V on head dim.
block_table = common_attn_metadata.block_table_tensor
seq_lens = common_attn_metadata.seq_lens.cpu()
query_lens = (
@@ -258,19 +260,21 @@ def _create_nvfp4_hnd_kv_cache(
# Gather context tokens from the bf16 cache using block table.
n_ctx_blocks = (ctx_len + block_size - 1) // block_size
blocks = block_table[i, :n_ctx_blocks]
# bf16_cache[:, kv_idx] is (num_blocks, block_size, num_kv_heads, head_size)
k_ctx = bf16_cache[blocks, 0].reshape(-1, num_kv_heads, head_size)[:ctx_len]
v_ctx = bf16_cache[blocks, 1].reshape(-1, num_kv_heads, head_size)[:ctx_len]
# bf16_cache is (B, 2*H, N, hs); split K and V head groups.
k_bf16, v_bf16 = bf16_cache[blocks].split(num_kv_heads, dim=1)
k_ctx = k_bf16.transpose(1, 2).reshape(-1, num_kv_heads, head_size)[:ctx_len]
v_ctx = v_bf16.transpose(1, 2).reshape(-1, num_kv_heads, head_size)[:ctx_len]
# Build slot mapping for these context tokens.
token_offsets = torch.arange(ctx_len, device=device)
block_indices = token_offsets // block_size
intra_offsets = token_offsets % block_size
slots = block_table[i, block_indices] * block_size + intra_offsets
# reshape_and_cache_flash expects (B, N, H, D) cache views.
torch.ops._C_cache_ops.reshape_and_cache_flash(
k_ctx,
v_ctx,
nvfp4_cache[:, 0],
nvfp4_cache[:, 1],
k_cache.transpose(1, 2),
v_cache.transpose(1, 2),
slots,
"nvfp4",
kv_scale_t,
@@ -289,10 +289,9 @@ async def test_send_kv_to_decode_aligns_consumer_regions_by_layer_metadata(
prefill_worker = prefill_connector.connector_worker
block_len = 4096
kv_half = block_len // 2
prefill_worker.kv_caches_base_addr = [0x1000]
prefill_worker.block_len_per_layer = [block_len]
prefill_worker.kv_block_len_per_layer = [kv_half]
prefill_worker.kv_block_len_per_layer = [block_len]
prefill_worker.registered_layer_names = ["model.layers.1.self_attn"]
prefill_worker.registered_layer_indices = [1]
@@ -321,7 +320,7 @@ async def test_send_kv_to_decode_aligns_consumer_regions_by_layer_metadata(
req_blocks={"d-req-layer-align": (transfer_id, [[20]])},
kv_caches_base_addr=[0xA000, 0xB000],
block_lens=[block_len, block_len],
kv_block_lens=[kv_half, kv_half],
kv_block_lens=[block_len, block_len],
registered_layer_names=[
"model.layers.0.self_attn",
"model.layers.1.self_attn",
@@ -338,15 +337,9 @@ async def test_send_kv_to_decode_aligns_consumer_regions_by_layer_metadata(
await prefill_worker.send_kv_to_decode(identity, mock_socket, xfer_meta)
src_ptrs, dst_ptrs, lengths = mock_send_blocks.call_args[0][1:]
assert src_ptrs == [
0x1000 + 10 * block_len,
0x1000 + 10 * block_len + kv_half,
]
assert dst_ptrs == [
0xB000 + 20 * block_len,
0xB000 + 20 * block_len + kv_half,
]
assert lengths == [kv_half, kv_half]
assert src_ptrs == [0x1000 + 10 * block_len]
assert dst_ptrs == [0xB000 + 20 * block_len]
assert lengths == [block_len]
sent_identity, sent_payload = mock_socket.send_multipart.call_args[0][0]
assert sent_identity == identity
@@ -832,9 +825,8 @@ async def test_kv_producer(monkeypatch):
prefill_worker = prefill_connector.connector_worker
prefill_worker.kv_caches_base_addr = [0x1000]
block_len = 4096
kv_half = block_len // 2
prefill_worker.block_len_per_layer = [block_len]
prefill_worker.kv_block_len_per_layer = [kv_half]
prefill_worker.kv_block_len_per_layer = [block_len]
prefill_worker.registered_layer_names = ["model.layers.0.self_attn"]
prefill_worker.registered_layer_indices = [0]
@@ -862,7 +854,7 @@ async def test_kv_producer(monkeypatch):
req_blocks={"d-req-1": (transfer_id, [[20, 21]])},
kv_caches_base_addr=[0x2000],
block_lens=[block_len],
kv_block_lens=[kv_half],
kv_block_lens=[block_len],
registered_layer_names=["model.layers.0.self_attn"],
registered_layer_indices=[0],
)
@@ -874,24 +866,18 @@ async def test_kv_producer(monkeypatch):
with patch.object(
prefill_worker, "_send_blocks", return_value=0
) as mock_send_blocks:
# With blocks-first layout, each block is virtually split
# into K and V halves, producing non-coalesced transfers.
def expected_split_transfers(src_base, dst_base, src_blocks, dst_blocks):
"""Build expected (src_ptrs, dst_ptrs, lengths) for
virtual-split K/V transfers."""
src_ptrs, dst_ptrs, lengths = [], [], []
for kv_offset in (0, kv_half):
for sb, db in zip(src_blocks, dst_blocks):
src_ptrs.append(src_base + sb * block_len + kv_offset)
dst_ptrs.append(dst_base + db * block_len + kv_offset)
lengths.append(kv_half)
return src_ptrs, dst_ptrs, lengths
def expected_transfers(src_base, dst_base, src_blocks, dst_blocks):
n = len(src_blocks)
return (
[src_base + src_blocks[0] * block_len],
[dst_base + dst_blocks[0] * block_len],
[n * block_len],
)
# Normal case: 2 blocks to 2 blocks
await prefill_worker.send_kv_to_decode(identity, mock_socket, xfer_meta)
src, dst, lens = expected_split_transfers(
0x1000, 0x2000, [10, 11], [20, 21]
)
src, dst, lens = expected_transfers(0x1000, 0x2000, [10, 11], [20, 21])
mock_send_blocks.assert_called_once_with(
"consumer-host:54321",
src,
@@ -923,7 +909,7 @@ async def test_kv_producer(monkeypatch):
# Worker processes the consumer's request
await prefill_worker.send_kv_to_decode(identity, mock_socket, xfer_meta)
# Verify transfer parameters are correct: 11 to 20
src, dst, lens = expected_split_transfers(0x1000, 0x2000, [11], [20])
src, dst, lens = expected_transfers(0x1000, 0x2000, [11], [20])
mock_send_blocks.assert_called_once_with(
"consumer-host:54321",
src,
@@ -1266,7 +1252,7 @@ async def test_kv_producer_heterogeneous_tp(monkeypatch, d_tp_size):
prefill_worker.kv_caches_base_addr = [0x1000]
prefill_worker.block_len_per_layer = [local_block_len]
prefill_worker.kv_block_len_per_layer = [local_block_len // 2]
prefill_worker.kv_block_len_per_layer = [local_block_len]
prefill_worker.registered_layer_names = ["model.layers.0.self_attn"]
prefill_worker.registered_layer_indices = [0]
@@ -1314,7 +1300,7 @@ async def test_kv_producer_heterogeneous_tp(monkeypatch, d_tp_size):
},
kv_caches_base_addr=[0x2000],
block_lens=[remote_block_len],
kv_block_lens=[remote_block_len // 2],
kv_block_lens=[remote_block_len],
registered_layer_names=["model.layers.0.self_attn"],
registered_layer_indices=[0],
)
@@ -1336,47 +1322,31 @@ async def test_kv_producer_heterogeneous_tp(monkeypatch, d_tp_size):
flat_remote = [b for g in remote_block_ids for b in g]
num_blocks = len(flat_local)
# With blocks-first layout, virtual split halves block
# lengths and doubles transfer regions (K + V).
local_kv_block_len = local_block_len // 2
remote_kv_block_len = remote_block_len // 2
assert len(src_ptrs) == num_blocks
assert len(dst_ptrs) == num_blocks
assert len(lengths) == num_blocks
assert len(src_ptrs) == 2 * num_blocks
assert len(dst_ptrs) == 2 * num_blocks
assert len(lengths) == 2 * num_blocks
# Compute expected offsets using kv_block_len
if d_tp_size <= P_TP_SIZE:
tp_ratio = P_TP_SIZE // d_tp_size
expected_src_off = 0
expected_dst_off = (P_TP_RANK % tp_ratio) * local_kv_block_len
expected_xfer_len = local_kv_block_len
expected_dst_off = (P_TP_RANK % tp_ratio) * local_block_len
expected_xfer_len = local_block_len
else:
ratio_abs = d_tp_size // P_TP_SIZE
expected_src_off = (d_rank % ratio_abs) * remote_kv_block_len
expected_src_off = (d_rank % ratio_abs) * remote_block_len
expected_dst_off = 0
expected_xfer_len = remote_kv_block_len
expected_xfer_len = remote_block_len
# First num_blocks entries are K region,
# next num_blocks are V region.
for region_idx in range(2):
local_region_base = 0x1000 + region_idx * local_kv_block_len
remote_region_base = 0x2000 + region_idx * remote_kv_block_len
for blk_idx, (lblk, rblk) in enumerate(
zip(flat_local, flat_remote)
):
idx = region_idx * num_blocks + blk_idx
assert src_ptrs[idx] == (
local_region_base
+ lblk * local_block_len
+ expected_src_off
)
assert dst_ptrs[idx] == (
remote_region_base
+ rblk * remote_block_len
+ expected_dst_off
)
assert lengths[idx] == expected_xfer_len
local_region_base = 0x1000
remote_region_base = 0x2000
for blk_idx, (lblk, rblk) in enumerate(zip(flat_local, flat_remote)):
assert src_ptrs[blk_idx] == (
local_region_base + lblk * local_block_len + expected_src_off
)
assert dst_ptrs[blk_idx] == (
remote_region_base + rblk * remote_block_len + expected_dst_off
)
assert lengths[blk_idx] == expected_xfer_len
# Verify successful response sent back to consumer
mock_socket.send_multipart.assert_called_once()
@@ -1068,6 +1068,65 @@ class TestNixlHandshake:
# whole block is moved.
worker.add_remote_agent(meta, remote_tp_size=1)
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
FakeNixlWrapper,
)
def test_hybrid_mamba_attention_remote_descs_use_packed_head_slices(
self, default_vllm_config, dist_init
):
worker = FakeNixlConnectorWorker(
create_vllm_config(), "engine", hand_shake_latency=0
)
remote_block_len = 2048
local_block_len = remote_block_len // 2
worker.block_len_per_layer = [local_block_len]
worker._region_is_mla = [False]
worker.num_blocks = 1
worker.num_regions = 1
worker._has_mamba = True
worker._mamba_ssm_size = (128, 256)
worker.transfer_topo = TransferTopology(
tp_rank=1,
tp_size=2,
block_size=worker.block_size,
engine_id=worker.engine_id,
is_mla=False,
is_mamba=True,
total_num_kv_heads=2,
attn_backends=worker.attn_backends,
tensor_shape=None,
)
assert worker.transfer_topo.virtually_split_kv_in_blocks
plan = MagicMock(
source_ranks_per_group=((0,), (0,)),
rank_offset_factor=1,
)
meta = MagicMock(
kv_caches_base_addr=[0x1000],
device_id=0,
num_blocks=1,
block_lens=[remote_block_len],
)
assert worker.get_backend_aware_kv_block_len(0, mamba_view=False) == (
local_block_len
)
assert (
worker.get_backend_aware_kv_block_len(0, first_split=True, mamba_view=True)
== worker._mamba_ssm_size[0]
)
assert (
worker.get_backend_aware_kv_block_len(0, first_split=False, mamba_view=True)
== worker._mamba_ssm_size[1]
)
assert worker._build_fa_remote(plan, meta, block_size_ratio=1) == [
(0x1000 + local_block_len, local_block_len, 0)
]
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
FakeNixlWrapper,
@@ -1691,13 +1750,6 @@ def _run_abort_timeout_test(llm: LLM, timeout: int):
reason="Attention backend FLASH_ATTN is not supported on ROCm",
),
),
pytest.param(
"ROCM_ATTN",
marks=pytest.mark.skipif(
not current_platform.is_rocm(),
reason="Attention backend ROCM_ATTN is only supported on ROCm",
),
),
"TRITON_ATTN",
],
)
@@ -1824,8 +1876,7 @@ def test_register_kv_caches(
test_shape = backend_cls.get_kv_cache_shape(
num_blocks=1, block_size=16, num_kv_heads=1, head_size=1
)
is_blocks_first = len(test_shape) == 5 and test_shape[0] == 1
virtually_split = is_blocks_first and not connector.prefer_cross_layer_blocks
is_blocks_first = len(test_shape) == 4 and test_shape[0] == 1
if connector.prefer_cross_layer_blocks:
with set_current_vllm_config(vllm_config):
@@ -1856,7 +1907,7 @@ def test_register_kv_caches(
]
expected_num_entries = 1
expected_blocks_count = num_blocks * (2 if virtually_split else 1)
expected_blocks_count = num_blocks
kv_caches = {"all-layers": cross_layers_kv_cache}
else:
@@ -1885,6 +1936,7 @@ def test_register_kv_caches(
unique_tensor.data_ptr(),
]
expected_num_entries = 2
expected_blocks_count = kv_cache_config.num_blocks * 2
else:
expected_tensor_size = (
shared_tensor[0].element_size() * shared_tensor[0].numel()
@@ -1896,7 +1948,7 @@ def test_register_kv_caches(
unique_tensor[1].data_ptr(),
]
expected_num_entries = 4
expected_blocks_count = kv_cache_config.num_blocks * 4
expected_blocks_count = kv_cache_config.num_blocks * 4
# Execute register_kv_caches
connector.register_kv_caches(kv_caches)
@@ -1930,10 +1982,7 @@ def test_register_kv_caches(
else:
num_blocks = kv_cache_config.num_blocks
if virtually_split:
expected_block_len = expected_tensor_size // num_blocks // 2
else:
expected_block_len = expected_tensor_size // num_blocks
expected_block_len = expected_tensor_size // num_blocks
for i, block_entry in enumerate(blocks_data):
block_start_addr, block_len, tp_rank = block_entry
@@ -18,8 +18,8 @@ class _FakeAttentionBackend:
block_size: int,
num_kv_heads: int,
head_size: int,
) -> tuple[int, int, int, int, int]:
return (2, num_blocks, num_kv_heads, block_size, head_size)
) -> tuple[int, int, int, int]:
return (num_blocks, num_kv_heads, block_size, 2 * head_size)
def _make_topology(
+7 -5
View File
@@ -795,9 +795,10 @@ def test_kv_cache_stride_order(monkeypatch, model_runner):
)
# TODO mla test
default_stride = tuple(range(5))
default_stride = tuple(range(len(expected_kv_cache_shape)))
non_default_stride = (*default_stride[1:], default_stride[0])
# Permutation that gets you back to expected kv shape
for test_stride in ((1, 4, 0, 2, 3), (0, 1, 2, 3, 4)):
for test_stride in (non_default_stride, default_stride):
def rnd_stride_order(
include_num_layers_dimension: bool = False, test_stride=test_stride
@@ -1300,9 +1301,10 @@ def test_hybrid_attention_mamba_tensor_shapes():
actual_kv = vllm_ctx[layer].kv_cache[kernel_block, :]
expected = attn_blocks_constant[i]
# Check K and V separately
assert torch.equal(actual_kv[0], expected)
assert torch.equal(actual_kv[1], expected)
# Packed layout: (num_kv_heads, block_size, 2*head_size). Every
# head in the block was filled with the same constant.
for head_idx in range(actual_kv.shape[0]):
assert torch.equal(actual_kv[head_idx], expected)
for layer in [layer_2, layer_3, layer_4, layer_5]:
for i, kv_block in enumerate(kv_blocks_for_mamba):
@@ -10,7 +10,12 @@ from typing import TYPE_CHECKING, Any, Literal, cast
import torch
from vllm.config import VllmConfig, get_current_vllm_config, get_layers_from_vllm_config
from vllm.config import (
VllmConfig,
get_current_vllm_config,
get_layers_from_vllm_config,
set_current_vllm_config,
)
from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory
from vllm.logger import init_logger
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
@@ -344,14 +349,15 @@ def get_current_attn_backends(
)
from vllm.v1.attention.selector import get_attn_backend
return [
get_attn_backend(
head_size=vllm_config.model_config.get_head_size(),
dtype=vllm_config.model_config.dtype,
kv_cache_dtype=vllm_config.cache_config.cache_dtype,
use_mla=vllm_config.model_config.use_mla,
)
]
with set_current_vllm_config(vllm_config):
return [
get_attn_backend(
head_size=vllm_config.model_config.get_head_size(),
dtype=vllm_config.model_config.dtype,
kv_cache_dtype=vllm_config.cache_config.cache_dtype,
use_mla=vllm_config.model_config.use_mla,
)
]
def get_current_attn_backend(
@@ -426,12 +432,16 @@ class TransferTopology:
head_size=1,
)
logger.debug("Test kv_cache_shape: %s", kv_cache_shape)
# Non-MLA backends caches have 5 dims [num_blocks, 2, H,N,D],
# we just mock num_blocks to 1 for the dimension check below.
# Hybrid SSM models assume a single blocks_first layout
self._is_kv_layout_blocks_first = self.is_mamba or (
len(kv_cache_shape) == 5 and kv_cache_shape[0] == 1
)
assert kv_cache_shape[0] == 1, (
"KV cache layout must be blocks-first; expected mocked "
f"num_blocks=1 in leading dim, got shape {kv_cache_shape}."
)
if not self.is_mla:
assert len(kv_cache_shape) == 4, (
"Attention KV cache layout must be standardized as "
"[num_blocks, num_kv_heads, block_size, content_size], "
f"got shape {kv_cache_shape}."
)
self._cross_layers_blocks = False
if self.tensor_shape is not None:
@@ -490,29 +500,19 @@ class TransferTopology:
# Layout properties
# ============================================================
@property
def is_kv_layout_blocks_first(self) -> bool:
return self._is_kv_layout_blocks_first
@property
def cross_layers_blocks(self) -> bool:
return self._cross_layers_blocks
@property
def virtually_split_kv_in_blocks(self) -> bool:
# Whether to logically split each block into K and V halves.
# Applies when K/V are interleaved within each block (blocks-first),
# but NOT when cross-layer blocks are used — cross-layer blocks have
# per-layer K/V interleaving (L0_K, L0_V, L1_K, L1_V, ...) so a
# simple half-split does not separate K from V.
return self._is_kv_layout_blocks_first and not self._cross_layers_blocks
@property
def split_k_and_v(self) -> bool:
# Whether to register regions for K and V separately (when present).
return not (
self._cross_layers_blocks or self.is_mla or self.is_kv_layout_blocks_first
)
# Whether to logically split each block into two separately-indexable
# sub-regions. With K and V packed into the content dim, an attention
# block transfers as a single unit — no K/V sub-split is needed. Only
# Mamba still needs this, to index its two state regions (conv/ssm)
# separately. Not applicable to cross-layer blocks (per-layer
# interleaving means a simple half-split does not separate the parts).
return self.is_mamba and not self._cross_layers_blocks
# ============================================================
# Common methods
@@ -616,8 +616,9 @@ class TransferTopology:
# Swap [2<>num_blocks] dims for hybrid SSM layout.
cache = cache.transpose(0, 1)
# Regular case: backends like FA register K/V in separate regions
return cache if self.split_k_and_v else [cache]
# K and V are packed into one tensor (content dim), so each layer
# registers as a single region.
return [cache]
def describe(self, remote_engine_id: EngineId, remote_pp_rank: int = 0) -> str:
"""One-line summary of transfer config for logging."""
@@ -7,6 +7,7 @@ from typing import NamedTuple
import torch
from vllm.v1.kv_cache_interface import (
AttentionSpec,
KVCacheConfig,
KVCacheSpec,
MLAAttentionSpec,
@@ -56,6 +57,11 @@ def is_mla_cache_layer(
return isinstance(spec, (MLAAttentionSpec, SlidingWindowMLASpec))
def _content_packed_dim(spec: AttentionSpec) -> int:
head_size_v = getattr(spec, "head_size_v", spec.head_size)
return spec.head_size + head_size_v
def _spec_dim_matches(value: int, expected: int | None) -> bool:
return expected is None or value == expected
@@ -223,6 +229,30 @@ def get_layer_transfer_geometry(
split_kv_regions=False,
)
if (
not is_mla_cache
and isinstance(spec, AttentionSpec)
and len(shape) == 4
and shape[1] == spec.num_kv_heads
and shape[2] == spec.block_size
and shape[3] == _content_packed_dim(spec)
):
num_blocks, num_kv_heads, block_size, packed_dim = shape
slot_size_bytes = num_kv_heads * packed_dim * element_size
block_len = block_size * slot_size_bytes
return LayerTransferGeometry(
num_blocks=num_blocks,
block_size=block_size,
block_len=block_len,
slot_size_bytes=slot_size_bytes,
block_stride=stride[0],
local_kv_stride=None,
remote_kv_stride=None,
transfers_per_block=1,
regions_per_block=1,
split_kv_regions=False,
)
cache_kind = "MLA" if is_mla_cache else "K/V"
raise ValueError(
f"Unsupported MoRIIO {cache_kind} cache shape for layer "
@@ -216,12 +216,10 @@ class NixlBaseConnectorWorker:
if n_regions == 0 or self.num_regions == 0:
return [False] * num_fa_descs
nblk = num_fa_descs // self.num_regions
virtually_split = self.transfer_topo.virtually_split_kv_in_blocks
flags: list[bool] = []
for i in range(n_regions):
replicated = self._is_region_replicated(i)
num_streams = 1 if replicated or not virtually_split else 2
flags.extend([replicated] * (num_streams * nblk))
flags.extend([replicated] * nblk)
assert len(flags) == num_fa_descs, (
f"FA desc flags {len(flags)} != num_fa_descs {num_fa_descs}"
)
@@ -712,31 +710,31 @@ class NixlBaseConnectorWorker:
NOT directly supported by NIXL (e.g., tpu)
"""
xfer_buffers: dict[str, torch.Tensor] = {}
inv_order = [0, 1, 3, 2, 4]
try:
for layer_name, kv_cache in kv_caches.items():
kv_shape = kv_cache.shape
kv_dtype = kv_cache.dtype
permute_shape = False
if (
self.kv_cache_layout == "NHD"
and self.vllm_config.kv_transfer_config is not None
and self.vllm_config.kv_transfer_config.enable_permute_local_kv
):
logger.info_once(
"'enable_permute_local_kv' flag is enabled while "
"device KV Layout is NHD. Init host buffer with"
" HND to better support Decode/Prefill TP_ratio > 1."
)
# Since NHD will not support Decode/Prefill TP_ratio > 1,
# we can leverage host_buffer for permute
self.host_buffer_kv_cache_layout = "HND"
kv_shape = (
tuple(kv_shape[i] for i in inv_order)
if not self.use_mla
else kv_shape
)
permute_shape = not self.use_mla
inv_order = (0, 2, 1, 3)
if not self.use_mla:
assert kv_cache.ndim == 4
if self.kv_cache_layout == "NHD":
if self.kv_transfer_config.enable_permute_local_kv:
logger.info_once(
"'enable_permute_local_kv' flag is enabled while "
"device KV Layout is NHD. Init host buffer with"
" HND to better support Decode/Prefill TP_ratio > 1."
)
# Since NHD will not support Decode/Prefill TP_ratio > 1,
# we can leverage host_buffer for permute.
self.host_buffer_kv_cache_layout = "HND"
else:
# Packed KV layout is logical (B, H, N, 2*D). Allocate
# (B, N, H, 2*D) and view it as logical (B, H, N, 2*D)
# so raw NIXL transfers see NHD physical strides.
kv_shape = tuple(kv_shape[i] for i in inv_order)
permute_shape = True
xfer_buffers[layer_name] = torch.empty(
kv_shape, dtype=kv_dtype, device="cpu"
@@ -1151,9 +1149,7 @@ class NixlBaseConnectorWorker:
f"backend={self.backend_name}, "
"all_backends="
f"{[backend.get_name() for backend in self.attn_backends]}, "
f"kv_cache_layout={self.kv_cache_layout}, "
"blocks_first="
f"{self.transfer_topo.is_kv_layout_blocks_first}"
f"kv_cache_layout={self.kv_cache_layout}"
)
# Need to make sure the device ID is non-negative for NIXL,
@@ -1184,22 +1180,6 @@ class NixlBaseConnectorWorker:
regions_per_layer = self.num_regions // num_local_layers
self._remote_region_offset = regions_per_layer * start_layer
if self.transfer_topo.virtually_split_kv_in_blocks:
# NOTE (NickLucche) When FlashInfer is used, memory is registered
# with joint KV for each block. This minimizes the overhead in
# registerMem allowing faster descs queries. In order to be able to
# split on kv_heads dim as required by heterogeneous TP, one must
# be able to index K/V separately. Hence we double the number
# of 'virtual' regions here and halve `block_len` below.
# Similarly for Mamba layers, we register SSM+Conv as a single region and
# then duplicate it logically to be able to index SSM/Conv separately.
# Exception: key-only REPLICATE regions (MLA) have no V half, so
# they contribute a single desc stream and are not doubled.
self.num_regions = sum(
1 if self._is_region_replicated(i) else 2
for i in range(len(self._region_is_mla))
)
# Total local FA descriptors (boundary between FA and mamba descs).
self.num_descs = self.num_regions * self.num_blocks
@@ -1363,22 +1343,6 @@ class NixlBaseConnectorWorker:
block_offset = block_id * page_stride
addr = base_addr + block_offset
result.append((addr, kv_block_len, self.device_id))
if (
self.transfer_topo.virtually_split_kv_in_blocks
and not self._is_region_replicated(i)
):
# Separate and interleave K/V regions to maintain the same
# descs ordering. This is needed for selecting contiguous heads
# when split across TP ranks. (Skipped for key-only REPLICATE.)
second_split = self.get_backend_aware_kv_block_len(
layer_idx=i, first_split=False, mamba_view=False
)
for block_id in range(num_blocks):
block_offset = block_id * page_stride
addr = base_addr + block_offset
v_addr = addr + kv_block_len
result.append((v_addr, second_split, self.device_id))
return result
def _build_fa_remote(
@@ -1423,20 +1387,6 @@ class NixlBaseConnectorWorker:
# tp rank of size local_block_len.
addr = base_addr + block_offset + rank_offset
result.append((addr, local_block_len, nixl_agent_meta.device_id))
emits_v = self.transfer_topo.virtually_split_kv_in_blocks and not replicated
if emits_v:
# With FlashInfer index V separately to allow head splitting.
second_split = self.get_backend_aware_kv_block_len(
layer_idx=i, first_split=False, mamba_view=False
)
second_split = second_split // num_reads
for block_id in range(num_blocks):
block_offset = block_id * page_size
addr = base_addr + block_offset + rank_offset
# Hop over the first split of remote page, K, to read V.
v_addr = addr + nixl_agent_meta.block_lens[i] // 2
result.append((v_addr, second_split, nixl_agent_meta.device_id))
return result
def register_local_xfer_handler(
@@ -1927,24 +1877,18 @@ class NixlBaseConnectorWorker:
block_size_ratio,
)
split_k_and_v = self.transfer_topo.split_k_and_v
for block_ids in block_ids_list:
indices = torch.tensor(block_ids, device=self.device_type, dtype=torch.long)
for _, cache_or_caches in self.device_kv_caches.items():
cache_list = cache_or_caches if split_k_and_v else [cache_or_caches]
for cache in cache_list:
if self.enable_permute_local_kv and block_size_ratio > 1:
kv_postprocess_blksize_and_layout_on_receive(
cache, indices, block_size_ratio
)
elif self.enable_permute_local_kv:
kv_postprocess_layout_on_receive(cache, indices)
else:
kv_postprocess_blksize_on_receive(
cache, indices, block_size_ratio
)
for cache in self.device_kv_caches.values():
if self.enable_permute_local_kv and block_size_ratio > 1:
kv_postprocess_blksize_and_layout_on_receive(
cache, indices, block_size_ratio
)
elif self.enable_permute_local_kv:
kv_postprocess_layout_on_receive(cache, indices)
else:
kv_postprocess_blksize_on_receive(cache, indices, block_size_ratio)
def post_process_device_kv_on_receive_heterogeneous_attn(
self, block_ids: list[int]
@@ -2398,12 +2342,10 @@ class NixlBaseConnectorWorker:
|1st_split-2nd_split| |1st_split-2nd_split |
"""
assert self.transfer_topo is not None
virtually_split = self.transfer_topo.virtually_split_kv_in_blocks
if virtually_split and mamba_view:
if self.transfer_topo.virtually_split_kv_in_blocks and mamba_view:
block_len = self._mamba_ssm_size[not first_split]
else:
half_block = virtually_split and not self._is_region_replicated(layer_idx)
block_len = self.block_len_per_layer[layer_idx] // (2 if half_block else 1)
block_len = self.block_len_per_layer[layer_idx]
return block_len
def get_kv_connector_stats(self) -> KVConnectorStats | None:
@@ -8,7 +8,8 @@ equal the sparse block size (128), so one selected block maps to exactly one
page.
Main K/V cache layout (vLLM):
``(num_blocks, 2, 128, num_kv_heads, head_dim)`` K=[:,0] V=[:,1]
``(num_blocks, num_kv_heads, 128, 2 * head_dim)``
K=[..., :head_dim] V=[..., head_dim:]
Only the paths MiniMax M3 uses are implemented: no attention sink, base-2
(exp2/log2) softmax. The decode kernels use split-K (flash-decoding) over the
@@ -50,7 +51,7 @@ _FP8_DTYPES = (
@triton.jit(do_not_specialize_on_alignment=["seq_lens", "prefix_lens"])
def _gqa_sparse_fwd_kernel(
q_ptr, # [total_q, num_heads, head_dim]
kv_cache_ptr, # main cache: [num_blocks, 2, 128, num_kv_heads, head_dim]
kv_cache_ptr, # main cache: [num_blocks, num_kv_heads, 128, 2*head_dim]
t_ptr, # topk_idx: [num_kv_heads, total_q, topk]
o_ptr, # [total_q, num_heads, head_dim]
block_table_ptr, # [num_reqs, max_blocks]
@@ -68,9 +69,8 @@ def _gqa_sparse_fwd_kernel(
stride_qh,
stride_qd,
stride_kv_blk,
stride_kv_kv,
stride_kv_pos,
stride_kv_h,
stride_kv_pos,
stride_kv_d,
stride_th,
stride_tn,
@@ -140,9 +140,8 @@ def _gqa_sparse_fwd_kernel(
k = tl.load(
kv_cache_ptr
+ page * stride_kv_blk
+ 0 * stride_kv_kv
+ off_n[None, :] * stride_kv_pos
+ pid_kh * stride_kv_h
+ off_n[None, :] * stride_kv_pos
+ off_d[:, None] * stride_kv_d,
mask=d_mask[:, None] & pos_mask[None, :],
other=0.0,
@@ -162,10 +161,9 @@ def _gqa_sparse_fwd_kernel(
v = tl.load(
kv_cache_ptr
+ page * stride_kv_blk
+ 1 * stride_kv_kv
+ off_n[:, None] * stride_kv_pos
+ pid_kh * stride_kv_h
+ off_d[None, :] * stride_kv_d,
+ off_n[:, None] * stride_kv_pos
+ (head_dim + off_d[None, :]) * stride_kv_d,
mask=pos_mask[:, None] & d_mask[None, :],
other=0.0,
)
@@ -206,7 +204,7 @@ def _gqa_sparse_fwd_kernel(
@triton.jit(do_not_specialize=["decode_query_len"])
def _gqa_sparse_decode_kernel(
q_ptr, # [total_q, num_heads, head_dim]
kv_cache_ptr, # main cache: [num_blocks, 2, 128, num_kv_heads, head_dim]
kv_cache_ptr, # main cache: [num_blocks, num_kv_heads, 128, 2*head_dim]
t_ptr, # topk_idx: [num_kv_heads, total_q, topk]
o_ptr, # partial out: [NUM_TOPK_CHUNKS, total_q, num_heads, head_dim]
lse_ptr, # partial lse (log2): [NUM_TOPK_CHUNKS, total_q, num_heads]
@@ -222,9 +220,8 @@ def _gqa_sparse_decode_kernel(
stride_qh,
stride_qd,
stride_kv_blk,
stride_kv_kv,
stride_kv_pos,
stride_kv_h,
stride_kv_pos,
stride_kv_d,
stride_th,
stride_tn,
@@ -300,9 +297,8 @@ def _gqa_sparse_decode_kernel(
k = tl.load(
kv_cache_ptr
+ page * stride_kv_blk
+ 0 * stride_kv_kv
+ off_n[None, :] * stride_kv_pos
+ pid_kh * stride_kv_h
+ off_n[None, :] * stride_kv_pos
+ off_d[:, None] * stride_kv_d,
mask=d_mask[:, None] & pos_mask[None, :],
other=0.0,
@@ -319,10 +315,9 @@ def _gqa_sparse_decode_kernel(
v = tl.load(
kv_cache_ptr
+ page * stride_kv_blk
+ 1 * stride_kv_kv
+ off_n[:, None] * stride_kv_pos
+ pid_kh * stride_kv_h
+ off_d[None, :] * stride_kv_d,
+ off_n[:, None] * stride_kv_pos
+ (head_dim + off_d[None, :]) * stride_kv_d,
mask=pos_mask[:, None] & d_mask[None, :],
other=0.0,
)
@@ -418,7 +413,7 @@ def _merge_topk_attn_out_kernel(
@torch.no_grad()
def minimax_m3_sparse_attn(
q: torch.Tensor, # [total_q, num_heads, head_dim]
kv_cache: torch.Tensor, # [num_blocks, 2, 128, num_kv_heads, head_dim]
kv_cache: torch.Tensor, # [num_blocks, num_kv_heads, 128, 2*head_dim]
topk_idx: torch.Tensor, # [num_kv_heads, total_q, topk]
block_table: torch.Tensor, # [batch, max_blocks]
cu_seqlens_q: torch.Tensor, # [batch+1] int32
@@ -459,7 +454,6 @@ def minimax_m3_sparse_attn(
kv_cache.stride(1),
kv_cache.stride(2),
kv_cache.stride(3),
kv_cache.stride(4),
topk_idx.stride(0),
topk_idx.stride(1),
topk_idx.stride(2),
@@ -476,7 +470,7 @@ def minimax_m3_sparse_attn(
@torch.no_grad()
def minimax_m3_sparse_attn_decode(
q: torch.Tensor, # [total_q, num_heads, head_dim]
kv_cache: torch.Tensor, # [num_blocks, 2, 128, num_kv_heads, head_dim]
kv_cache: torch.Tensor, # [num_blocks, num_kv_heads, 128, 2*head_dim]
topk_idx: torch.Tensor, # [num_kv_heads, total_q, topk]
block_table: torch.Tensor, # [num_reqs, max_blocks]
seq_lens: torch.Tensor, # [num_reqs] int32
@@ -529,7 +523,6 @@ def minimax_m3_sparse_attn_decode(
kv_cache.stride(1),
kv_cache.stride(2),
kv_cache.stride(3),
kv_cache.stride(4),
topk_idx.stride(0),
topk_idx.stride(1),
topk_idx.stride(2),
@@ -104,20 +104,25 @@ class MiniMaxM3SparseBackend(AttentionBackend):
head_size: int,
cache_dtype_str: str = "auto",
) -> tuple[int, ...]:
return (num_blocks, 2, block_size, num_kv_heads, head_size)
# K and V are packed into the content dim: logical (B, H, N, 2*hs).
return (num_blocks, num_kv_heads, block_size, 2 * head_size)
@staticmethod
def get_kv_cache_stride_order(
include_num_layers_dimension: bool = False,
) -> tuple[int, ...]:
# Permutation from get_kv_cache_shape to the actual memory layout.
# `stride_order` indicates the permutation that gets us from
# `get_kv_cache_shape` (logical (B, H, N, 2*hs)) to the actual memory
# layout we want.
if include_num_layers_dimension:
raise NotImplementedError # no cross-layer KV blocks in M3
cache_layout = get_kv_cache_layout()
if cache_layout == "NHD":
stride_order = (0, 1, 2, 3, 4)
# (num_blocks, block_size, num_kv_heads, 2*head_size)
stride_order = (0, 2, 1, 3)
elif cache_layout == "HND":
stride_order = (0, 1, 3, 2, 4)
# (num_blocks, num_kv_heads, block_size, 2*head_size)
stride_order = (0, 1, 2, 3)
else:
raise ValueError(f"Unknown cache layout format {cache_layout}.")
return stride_order
@@ -79,8 +79,7 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
# strided view directly (topK stays innermost-contiguous).
prefill_topk = topk[nd:num_tokens].transpose(0, 1)
qp = q[nd:]
k_cache = kv_cache[:, 0].transpose(1, 2)
v_cache = kv_cache[:, 1].transpose(1, 2)
k_cache, v_cache = kv_cache.split(self.head_size, dim=-1)
k2q_row_ptr, k2q_q_indices, schedule = build_k2q_csr(
prefill_topk,
p.cu_seqlens_q,
+16 -2
View File
@@ -427,8 +427,8 @@ def _get_backend_priorities(
]
backends = []
# ROCM_ATTN uses (2, num_blocks, ...) KV cache layout which is
# incompatible with KV connectors that require blocks-first layout.
# Keep ROCM_ATTN disabled for KV connectors until connector transfer
# semantics are validated for its asymmetric native K/V cache views.
if not use_kv_connector:
backends.append(AttentionBackendEnum.ROCM_ATTN)
if rocm_aiter_ops.is_mha_enabled():
@@ -511,6 +511,20 @@ class RocmPlatform(Platform):
attn_selector_config.use_sparse,
attn_selector_config.use_kv_connector,
)
from vllm.config import get_current_vllm_config_or_none
vllm_config = get_current_vllm_config_or_none()
is_encoder_decoder = (
getattr(getattr(vllm_config, "model_config", None), "attn_type", None)
== "encoder_decoder"
)
# ROCM_ATTN still uses a legacy attention layout (KV is the outer
# dimension) that is incompatible with the encoder backend layouts. The
# encoder and decoder need the layouts to match. This is currently
# enforced implicitly.
# TODO: Make this explicit in the selector in a future PR.
if is_encoder_decoder and AttentionBackendEnum.ROCM_ATTN in backend_priorities:
backend_priorities.remove(AttentionBackendEnum.ROCM_ATTN)
for priority, backend in enumerate(backend_priorities):
try:
backend_class = backend.get_class()
+11 -45
View File
@@ -416,26 +416,24 @@ def nvfp4_kv_cache_full_dim(head_size: int) -> int:
return head_size // 2 + head_size // 16
def _nvfp4_split_data_scale(
def nvfp4_split_data_scale(
kv_side: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Split a single NVFP4 KV-side buffer into data and scale views.
"""Split one side (K or V) of an NVFP4 KV cache into data and scale.
The input is a 4D tensor for one KV side (K or V) whose last
dimension is ``full_dim = data_dim + scale_dim``. The physical
layout within each side is [data | scale], both packed contiguously.
The input is a 4D uint8 tensor whose last dimension is
``full_dim = data_dim + scale_dim``. The physical layout within each
side is ``[data | scale]``, both packed contiguously.
The caller is responsible for slicing K and V from the combined cache
first (e.g. ``kv_cache.split(num_kv_heads, dim=1)``).
Args:
kv_side: 4D uint8 tensor with shape
``(num_pages, dim_1, dim_2, full_dim)``.
May be in any permutation order (NHD or HND).
kv_side: 4D uint8 tensor ``(B, H, N, full_dim)``.
Returns:
``(data, scale)`` where
``data`` is a uint8 view with shape
``(num_pages, dim_1, dim_2, data_dim)``.
``scale`` is a float8_e4m3fn view with shape
``(num_pages, dim_1, dim_2, scale_dim)``.
``(data, scale)`` where *data* is uint8 and *scale* is
float8_e4m3fn, both views of the same storage.
"""
num_pages = kv_side.shape[0]
dim_1, dim_2 = kv_side.shape[1], kv_side.shape[2]
@@ -468,38 +466,6 @@ def _nvfp4_split_data_scale(
return data, scale
def nvfp4_kv_cache_split_views(kv_cache: torch.Tensor) -> tuple[tuple, tuple]:
"""Split an NVFP4 KV cache tensor into data and scale views.
Accepts either a 5D tensor ``(num_pages, 2, dim_2, dim_3, full_dim)``
or a 4D single-side tensor ``(num_pages, dim_2, dim_3, full_dim)``.
Per-page layout: [K_data | K_scale | V_data | V_scale].
Each KV side is self-contained (data followed by its scale), so the
5D case simply splits each side independently.
The returned views are in the same dim order as the input (NHD or
HND), so callers get views matching whichever order they passed in.
Args:
kv_cache: 5D or 4D uint8 tensor where the last dimension is
``full_dim = data_dim + scale_dim = 9 * head_size / 16``.
Returns:
For 5D input:
``(k_data, v_data), (k_scale, v_scale)``
For 4D input (single KV side):
``(data,), (scale,)``
"""
if kv_cache.dim() == 4:
data, scale = _nvfp4_split_data_scale(kv_cache)
return (data,), (scale,)
k_data, k_scale = _nvfp4_split_data_scale(kv_cache[:, 0])
v_data, v_scale = _nvfp4_split_data_scale(kv_cache[:, 1])
return (k_data, v_data), (k_scale, v_scale)
def create_kv_caches_with_random_flash(
num_blocks: int,
block_size: int,
+6 -6
View File
@@ -122,11 +122,11 @@ class AttentionBackend(ABC):
) -> tuple[int, ...]:
"""
Get the physical (memory layout) ordering of the kv cache dimensions.
e.g. if the KV cache shape is
[2, num_blocks, block_size, num_heads, head_size],
and get_kv_cache_stride_order returns (1, 3, 0, 2, 4) then the physical
Standard attention backends pack K and V into the content dim, giving
the logical shape [num_blocks, num_heads, block_size, 2 * head_size].
e.g. if get_kv_cache_stride_order returns (0, 2, 1, 3) then the physical
ordering of dimensions is
[num_blocks, num_heads, 2, block_size, head_size].
[num_blocks, block_size, num_heads, 2 * head_size].
If this function is unimplemented / raises NotImplementedError,
the physical layout of the KV cache will match the logical shape.
@@ -135,9 +135,9 @@ class AttentionBackend(ABC):
include_num_layers_dimension: if True, includes an additional
num_layers dimension, which is assumed to be prepended
to the logical KV cache shape.
With the above example, a return value (2, 4, 0, 1, 3, 5)
With the above example, a return value (1, 0, 3, 2, 4)
corresponds to
[num_blocks, num_heads, num_layers, 2, block_size, head_size].
[num_blocks, num_layers, block_size, num_heads, 2 * head_size].
If an additional dimension is NOT included in the returned
tuple, the physical layout will not include a layers dimension.
+18 -13
View File
@@ -133,25 +133,29 @@ class FlashAttentionBackend(AttentionBackend):
) -> tuple[int, ...]:
if block_size % 16 != 0:
raise ValueError("Block size must be a multiple of 16.")
return (num_blocks, 2, block_size, num_kv_heads, head_size)
# K and V are packed into the content dim: logical (B, H, N, 2*D).
return (num_blocks, num_kv_heads, block_size, 2 * head_size)
@staticmethod
def get_kv_cache_stride_order(
include_num_layers_dimension: bool = False,
) -> tuple[int, ...]:
# `stride_order` indicates the permutation that gets
# us from `get_kv_cache_shape` to the actual memory layout we want.
# `stride_order` indicates the permutation that gets us from
# `get_kv_cache_shape` (logical (B, H, N, 2*D)) to the actual memory
# layout we want.
cache_layout = get_kv_cache_layout()
if cache_layout == "NHD" and include_num_layers_dimension:
# (num_blocks, num_layers, 2, block_size, num_kv_heads, head_size)
return (1, 0, 2, 3, 4, 5)
# (num_blocks, num_layers, block_size, num_kv_heads, 2*head_size)
return (1, 0, 3, 2, 4)
elif cache_layout == "NHD":
stride_order = (0, 1, 2, 3, 4)
# (num_blocks, block_size, num_kv_heads, 2*head_size)
stride_order = (0, 2, 1, 3)
elif cache_layout == "HND" and include_num_layers_dimension:
# (num_blocks, num_kv_heads, num_layers, 2, block_size, head_size)
return (1, 4, 0, 2, 3, 5)
# (num_blocks, num_kv_heads, num_layers, block_size, 2*head_size)
return (1, 2, 0, 3, 4)
elif cache_layout == "HND":
stride_order = (0, 1, 3, 2, 4)
# (num_blocks, num_kv_heads, block_size, 2*head_size)
stride_order = (0, 1, 2, 3)
else:
raise ValueError(f"Unknown cache layout format {cache_layout}.")
return stride_order
@@ -819,7 +823,7 @@ class FlashAttentionImpl(AttentionImpl):
key: shape = [num_tokens, num_kv_heads, head_size]
value: shape = [num_tokens, num_kv_heads, head_size]
kv_cache: shape =
[num_blocks, 2, block_size, num_kv_heads, head_size]
[num_blocks, num_kv_heads, block_size, 2 * head_size]
attn_metadata: Metadata for attention.
Returns:
shape = [num_tokens, num_heads * head_size]
@@ -866,8 +870,8 @@ class FlashAttentionImpl(AttentionImpl):
layer,
)
# For decoder and cross-attention, use KV cache as before
key_cache, value_cache = kv_cache.unbind(1)
# (B, H, N, 2*D) -> ((B, N, H, D), (B, N, H, D))
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
# Fix degenerate strides on size-1 dims (e.g. num_kv_heads=1 with TP).
# FA3/4 on H100+ uses TMA, which requires ≥16-byte stride alignment.
# See vllm.utils.torch_utils.canonicalize_singleton_dim_strides.
@@ -1075,7 +1079,8 @@ class FlashAttentionImpl(AttentionImpl):
# Scatter write into the KV cache using slot_mapping indices.
# No TMA kernel is invoked here, so stride canonicalization is not needed.
key_cache, value_cache = kv_cache.unbind(1)
# (B, H, N, 2*D) -> ((B, N, H, D), (B, N, H, D))
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
# Reshape the input keys and values and store them in the cache.
# Skip this if sharing KV cache with an earlier attention layer.
+19 -25
View File
@@ -83,10 +83,12 @@ class FlashAttentionDiffKVBackend(FlashAttentionBackend):
) -> tuple[int, ...]:
if block_size % 16 != 0:
raise ValueError("Block size must be a multiple of 16.")
# Logical (blocks-first, head-major) layout: K and V (with their
# different head sizes) packed in the content dim.
return (
num_blocks,
block_size,
num_kv_heads,
block_size,
head_size + FlashAttentionDiffKVBackend.head_size_v,
)
@@ -94,21 +96,22 @@ class FlashAttentionDiffKVBackend(FlashAttentionBackend):
def get_kv_cache_stride_order(
include_num_layers_dimension: bool = False,
) -> tuple[int, ...]:
# `stride_order` indicates the permutation that gets
# us from `get_kv_cache_shape` to the actual memory layout we want.
# `stride_order` indicates the permutation that gets us from
# `get_kv_cache_shape` (logical (B, H, N, C_k+C_v)) to the actual
# memory layout we want.
cache_layout = get_kv_cache_layout()
if cache_layout == "NHD" and include_num_layers_dimension:
# (num_blocks, num_layers, block_size,
# num_kv_heads, head_size + head_size_v)
return (1, 0, 2, 3, 4)
# (num_blocks, num_layers, block_size, num_kv_heads, C_k+C_v)
return (1, 0, 3, 2, 4)
elif cache_layout == "NHD":
stride_order = (0, 1, 2, 3)
elif cache_layout == "HND" and include_num_layers_dimension:
# (num_blocks, num_kv_heads, num_layers,
# block_size, head_size + head_size_v)
return (1, 3, 0, 2, 4)
elif cache_layout == "HND":
# (num_blocks, block_size, num_kv_heads, C_k+C_v)
stride_order = (0, 2, 1, 3)
elif cache_layout == "HND" and include_num_layers_dimension:
# (num_blocks, num_kv_heads, num_layers, block_size, C_k+C_v)
return (1, 2, 0, 3, 4)
elif cache_layout == "HND":
# (num_blocks, num_kv_heads, block_size, C_k+C_v)
stride_order = (0, 1, 2, 3)
else:
raise ValueError(f"Unknown cache layout format {cache_layout}.")
return stride_order
@@ -142,21 +145,14 @@ class FlashAttentionDiffKVImpl(FlashAttentionImpl):
# we use direct Q, K, V tensors without caching
return
# Unlike standard FlashAttn which splits kv_cache via unbind(0),
# DiffKV packs K and V into a single tensor along the last dim:
# kv_cache shape: [num_blocks, block_size, num_kv_heads,
# head_size_k + head_size_v]
# The triton kernel handles this combined layout directly.
#
# NOTE(woosuk): key and value are padded while slot_mapping is
# not padded. However, we don't need to do key[:num_actual_tokens]
# and value[:num_actual_tokens] because the reshape_and_cache_flash
# op uses the slot_mapping's shape to determine the number of
# actual tokens.
# (B, H, N, C) -> (B, N, H, C) for kernel compatibility.
triton_reshape_and_cache_flash_diffkv(
key,
value,
kv_cache,
kv_cache.transpose(1, 2),
slot_mapping,
self.kv_cache_dtype,
layer._k_scale,
@@ -229,10 +225,8 @@ class FlashAttentionDiffKVImpl(FlashAttentionImpl):
layer,
)
# For decoder and cross-attention, use KV cache as before
# Different head_size for K and V
key_cache = kv_cache[..., : self.head_size]
value_cache = kv_cache[..., self.head_size :]
# (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs))
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
# Fix degenerate strides on size-1 dims (e.g. num_kv_heads=1 with TP).
# FA3/4 on H100+ uses TMA, which requires ≥16-byte stride alignment.
# See vllm.utils.torch_utils.canonicalize_singleton_dim_strides.
+97 -43
View File
@@ -51,7 +51,7 @@ from vllm.utils.torch_utils import (
is_quantized_kv_cache,
is_strictly_contiguous,
nvfp4_kv_cache_full_dim,
nvfp4_kv_cache_split_views,
nvfp4_split_data_scale,
)
from vllm.v1.attention.backend import (
AttentionBackend,
@@ -111,9 +111,12 @@ def _trtllm_prefill_attn_kvfp8_dequant(
src_stride_page,
src_stride_kv,
src_stride_head,
src_stride_block,
src_stride_head_size,
DST_K_CACHE_STRIDE: tl.constexpr,
DST_KV_CACHE_STRIDE: tl.constexpr,
HEAD_STRIDE: tl.constexpr,
HEAD_SIZE: tl.constexpr,
NUM_KV_HEADS: tl.constexpr,
):
batch_idx = tl.program_id(0).to(tl.int64)
@@ -129,18 +132,25 @@ def _trtllm_prefill_attn_kvfp8_dequant(
v_scale_val = tl.load(v_scale_ptr)
mock_page_idx = batch_idx * block_table_stride + mock_block_table_idx + 1
head_offsets = tl.arange(0, HEAD_STRIDE)
logical_offsets = tl.arange(0, HEAD_STRIDE)
block_offsets = logical_offsets // HEAD_SIZE
head_size_offsets = logical_offsets % HEAD_SIZE
for h in range(NUM_KV_HEADS):
h_off = tl.cast(h, tl.int64)
# Read K from source (supports non-contiguous page/kv/head strides)
src_k = orig_page_num * src_stride_page + h_off * src_stride_head + head_offsets
src_k = (
orig_page_num * src_stride_page
+ h_off * src_stride_head
+ block_offsets * src_stride_block
+ head_size_offsets * src_stride_head_size
)
fp8_k = tl.load(kv_cache_ptr + src_k)
dequant_k = (fp8_k.to(tl.float32) * k_scale_val).to(dequant_dtype)
# Write K to contiguous mock cache
dst_k = mock_page_idx * DST_KV_CACHE_STRIDE + h * HEAD_STRIDE + head_offsets
dst_k = mock_page_idx * DST_KV_CACHE_STRIDE + h * HEAD_STRIDE + logical_offsets
tl.store(mock_kv_cache_ptr + dst_k, dequant_k)
# Read V from source (offset by src_stride_kv for the V half)
@@ -148,7 +158,8 @@ def _trtllm_prefill_attn_kvfp8_dequant(
orig_page_num * src_stride_page
+ src_stride_kv
+ h_off * src_stride_head
+ head_offsets
+ block_offsets * src_stride_block
+ head_size_offsets * src_stride_head_size
)
fp8_v = tl.load(kv_cache_ptr + src_v)
dequant_v = (fp8_v.to(tl.float32) * v_scale_val).to(dequant_dtype)
@@ -158,7 +169,7 @@ def _trtllm_prefill_attn_kvfp8_dequant(
mock_page_idx * DST_KV_CACHE_STRIDE
+ DST_K_CACHE_STRIDE
+ h * HEAD_STRIDE
+ head_offsets
+ logical_offsets
)
tl.store(mock_kv_cache_ptr + dst_v, dequant_v)
@@ -175,17 +186,14 @@ def trtllm_prefill_attn_kvfp8_dequant(
assert s[1] == 2
assert dequant_dtype in (torch.bfloat16, torch.float16)
# Logical source layout is (B, 2, H, N, D). The tensor may be a
# non-contiguous view, so the Triton kernel indexes it with actual strides.
strides = kv_cache.stride()
num_kv_heads, block_size, head_size = s[2], s[3], s[4]
head_stride = block_size * head_size
k_cache_stride = num_kv_heads * head_stride
kv_cache_stride = k_cache_stride * s[1]
strides = kv_cache.stride()
assert strides[3] == head_size and strides[4] == 1, (
"For kv cache layouts, (block_size, head_size) "
f"dimensions must be contiguous, got strides {strides}"
)
new_s = (batch_size * num_of_page_per_token + 1, s[1], s[2], s[3], s[4])
# mock kv cache contains just the pages needed by this prefill
mock_kv_cache = torch.empty(new_s, dtype=dequant_dtype, device=kv_cache.device)
@@ -207,9 +215,12 @@ def trtllm_prefill_attn_kvfp8_dequant(
strides[0],
strides[1],
strides[2],
strides[3],
strides[4],
k_cache_stride,
kv_cache_stride,
head_stride,
head_size,
num_kv_heads,
)
return mock_kv_cache, mock_block_table
@@ -286,7 +297,7 @@ class BatchDCPPrefillWrapper:
self,
layer: torch.nn.Module,
prefill_query: torch.Tensor,
kv_cache_permute: torch.Tensor,
kv_cache_tuple: tuple[torch.Tensor, torch.Tensor],
key: torch.Tensor,
value: torch.Tensor,
out: torch.Tensor,
@@ -296,7 +307,7 @@ class BatchDCPPrefillWrapper:
)
output_context_tmp, lse_context_tmp = self._context.run(
prefill_query_across_dcp,
kv_cache_permute,
kv_cache_tuple,
k_scale=layer._k_scale_float,
v_scale=layer._v_scale_float,
return_lse=True,
@@ -386,28 +397,31 @@ class FlashInferBackend(AttentionBackend):
cache_dtype_str: str = "auto",
) -> tuple[int, ...]:
if cache_dtype_str == "nvfp4":
# Packed layout: fp4 data + fp8 block scales in last dim
last_dim = nvfp4_kv_cache_full_dim(head_size)
return (num_blocks, 2, block_size, num_kv_heads, last_dim)
return (num_blocks, 2, block_size, num_kv_heads, head_size)
full_dim = nvfp4_kv_cache_full_dim(head_size)
return (num_blocks, 2 * num_kv_heads, block_size, full_dim)
# Pack K and V in the content dim (B, H, N, 2*hs).
return (num_blocks, num_kv_heads, block_size, 2 * head_size)
@staticmethod
def get_kv_cache_stride_order(
include_num_layers_dimension: bool = False,
) -> tuple[int, ...]:
# `stride_order` indicates the permutation that gets us from
# `get_kv_cache_shape` to the actual memory layout we want.
# `get_kv_cache_shape` (logical (B, H, N, 2*hs)) to the actual memory
# layout we want.
cache_layout = get_kv_cache_layout()
if cache_layout == "NHD" and include_num_layers_dimension:
# (num_blocks, num_layers, 2, block_size, num_kv_heads, head_size)
return (1, 0, 2, 3, 4, 5)
# (num_blocks, num_layers, block_size, num_kv_heads, 2*head_size)
return (1, 0, 3, 2, 4)
elif cache_layout == "NHD":
stride_order = (0, 1, 2, 3, 4)
# (num_blocks, block_size, num_kv_heads, 2*head_size)
stride_order = (0, 2, 1, 3)
elif cache_layout == "HND" and include_num_layers_dimension:
# (num_blocks, 2, num_kv_heads, num_layers, block_size, head_size)
return (1, 2, 4, 0, 3, 5)
# (num_blocks, num_kv_heads, num_layers, block_size, 2*head_size)
return (1, 2, 0, 3, 4)
elif cache_layout == "HND":
stride_order = (0, 1, 3, 2, 4)
# (num_blocks, num_kv_heads, block_size, 2*head_size)
stride_order = (0, 1, 2, 3)
else:
raise ValueError(f"Unknown cache layout format {cache_layout}.")
return stride_order
@@ -1714,7 +1728,19 @@ class FlashInferImpl(AttentionImpl):
if attn_metadata.use_cascade:
# Cascade attention (rare case).
assert attn_metadata.cascade_wrapper is not None
output.copy_(attn_metadata.cascade_wrapper.run(query, kv_cache))
stride_order = FlashInferBackend.get_kv_cache_stride_order()
if self.is_kvcache_nvfp4:
kv_cache_views = tuple(
cache.permute(*stride_order)
for cache in kv_cache.split(self.num_kv_heads, dim=1)
)
else:
kv_perm = kv_cache.permute(*stride_order)
kv_cache_views = kv_perm.split(self.head_size, dim=-1)
kv_tuple = tuple(
canonicalize_singleton_dim_strides(cache) for cache in kv_cache_views
)
output.copy_(attn_metadata.cascade_wrapper.run(query, kv_tuple))
return output
# When using spec decoding, num_decodes can be < num_decode_tokens
@@ -1740,14 +1766,23 @@ class FlashInferImpl(AttentionImpl):
)
kv_cache_permute = fixed
# For NVFP4, the kv_cache last dim is full_dim (data + scale packed).
# Split into correctly-strided data and scale views.
# Split K/V — zero-copy views. NVFP4 stores K/V as separate head
# groups; other dtypes pack K/V in the content dim.
hs = self.head_size
nvfp4_kv_data = None
nvfp4_kv_block_scales = None
if self.is_kvcache_nvfp4:
nvfp4_kv_data, nvfp4_kv_block_scales = nvfp4_kv_cache_split_views(
kv_cache_permute
k_cache, v_cache = kv_cache.split(self.num_kv_heads, dim=1)
kv_cache_tuple = (
canonicalize_singleton_dim_strides(k_cache.permute(*stride_order)),
canonicalize_singleton_dim_strides(v_cache.permute(*stride_order)),
)
k_data, k_sf = nvfp4_split_data_scale(kv_cache_tuple[0])
v_data, v_sf = nvfp4_split_data_scale(kv_cache_tuple[1])
nvfp4_kv_data = (k_data, v_data)
nvfp4_kv_block_scales = (k_sf, v_sf)
else:
kv_cache_tuple = kv_cache_permute.split(hs, dim=-1)
use_dcp = self.dcp_world_size > 1
@@ -1786,7 +1821,7 @@ class FlashInferImpl(AttentionImpl):
prefill_wrapper.run(
layer,
prefill_query,
kv_cache_permute,
kv_cache_tuple,
key[num_decode_tokens:],
value[num_decode_tokens:],
out=output[num_decode_tokens:],
@@ -1803,7 +1838,9 @@ class FlashInferImpl(AttentionImpl):
assert prefill_wrapper._causal == attn_metadata.causal
if self.is_kvcache_nvfp4:
kv_cache_permute = nvfp4_kv_data
kv_cache_for_fi = nvfp4_kv_data
else:
kv_cache_for_fi = kv_cache_tuple
kv_cache_sf = (
nvfp4_kv_block_scales if self.is_kvcache_nvfp4 else None
)
@@ -1821,7 +1858,7 @@ class FlashInferImpl(AttentionImpl):
prefill_wrapper.run(
prefill_query,
kv_cache_permute,
kv_cache_for_fi,
q_scale=layer._q_scale_float,
k_scale=layer._k_scale_float,
v_scale=layer._v_scale_float,
@@ -1888,8 +1925,7 @@ class FlashInferImpl(AttentionImpl):
# TRTLLM prefill attention does not support BF16 Q
# and fp8 kv cache. So to enable prefill attention
# with fp8 kv cache, we can construct a mock block
# and mock kv cache with BF16 KV involved in the prefill
#
# and mock kv cache with BF16 KV involved in the prefill.
kv_cache_permute = canonicalize_singleton_dim_strides(
kv_cache_permute
)
@@ -1901,15 +1937,21 @@ class FlashInferImpl(AttentionImpl):
"KV cache inner dims (block_size, head_size) must be "
f"contiguous, got strides {kv_strides}"
)
# fp8 uses (B, H, N, 2*hs); reshape to (B, 2, H, N, hs)
# for the dequant kernel — zero-copy view. The dequant
# kernel handles the interleaved K/V block stride.
B_kv, H_kv, N_kv = kv_cache_permute.shape[:3]
kv_cache_5d = kv_cache_permute.view(B_kv, H_kv, N_kv, 2, hs)
kv_cache_5d = kv_cache_5d.permute(0, 3, 1, 2, 4)
mock_kv_cache, mock_block_table = trtllm_prefill_attn_kvfp8_dequant(
kv_cache_permute,
kv_cache_5d,
block_tables_prefill,
layer._k_scale,
layer._v_scale,
attn_metadata.q_data_type_prefill,
)
else:
mock_kv_cache = kv_cache_permute
mock_kv_cache = kv_cache_tuple
mock_block_table = block_tables_prefill
trtllm_batch_context_with_kv_cache(
@@ -1957,7 +1999,9 @@ class FlashInferImpl(AttentionImpl):
assert decode_wrapper._sm_scale == self.scale
if self.is_kvcache_nvfp4:
kv_cache_permute = nvfp4_kv_data
kv_cache_for_fi = nvfp4_kv_data
else:
kv_cache_for_fi = kv_cache_tuple
kv_cache_sf = nvfp4_kv_block_scales if self.is_kvcache_nvfp4 else None
# NVFP4 kernel only supports FP8 output.
@@ -1980,7 +2024,7 @@ class FlashInferImpl(AttentionImpl):
)
decode_wrapper.run(
decode_query,
kv_cache_permute,
kv_cache_for_fi,
q_scale=layer._q_scale_float,
k_scale=layer._k_scale_float,
v_scale=layer._v_scale_float,
@@ -1997,7 +2041,7 @@ class FlashInferImpl(AttentionImpl):
else:
decode_wrapper.run(
decode_query,
kv_cache_permute,
kv_cache_for_fi,
q_scale=layer._q_scale_float,
k_scale=layer._k_scale_float,
v_scale=layer._v_scale_float,
@@ -2079,7 +2123,7 @@ class FlashInferImpl(AttentionImpl):
trtllm_batch_decode_with_kv_cache(
query=decode_query,
kv_cache=(
nvfp4_kv_data if self.is_kvcache_nvfp4 else kv_cache_permute
nvfp4_kv_data if self.is_kvcache_nvfp4 else kv_cache_tuple
),
workspace_buffer=workspace_buffer,
block_tables=block_tables_decode,
@@ -2119,8 +2163,18 @@ class FlashInferImpl(AttentionImpl):
# and value[:num_actual_tokens] because the reshape_and_cache_flash
# op uses the slot_mapping's shape to determine the number of
# actual tokens.
k_cache = kv_cache[:, 0]
v_cache = kv_cache[:, 1]
if self.is_kvcache_nvfp4:
# (B, 2*H, N, full_dim) -> ((B, N, H, full_dim),
# (B, N, H, full_dim));
# K heads first, then V heads.
k_cache, v_cache = kv_cache.transpose(1, 2).split(
self.num_kv_heads, dim=-2
)
else:
# (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs))
k_cache, v_cache = kv_cache.transpose(1, 2).split(
self.head_size, dim=-1
)
torch.ops._C_cache_ops.reshape_and_cache_flash(
key,
value,
+11 -7
View File
@@ -130,15 +130,16 @@ class FlexAttentionBackend(AttentionBackend):
head_size: int,
cache_dtype_str: str = "auto",
) -> tuple[int, ...]:
return (num_blocks, 2, block_size, num_kv_heads, head_size)
# K and V are packed into the content dim: logical (B, H, N, 2*hs).
return (num_blocks, num_kv_heads, block_size, 2 * head_size)
@staticmethod
def get_kv_cache_stride_order(
include_num_layers_dimension: bool = False,
) -> tuple[int, ...]:
if include_num_layers_dimension:
return (1, 0, 3, 2, 4, 5)
return (0, 2, 1, 3, 4)
return (1, 0, 3, 2, 4)
return (0, 2, 1, 3)
@staticmethod
def get_builder_cls() -> type["FlexAttentionMetadataBuilder"]:
@@ -1242,7 +1243,8 @@ class FlexAttentionImpl(AttentionImpl):
if self.attn_type == AttentionType.ENCODER_ONLY:
return
key_cache, value_cache = kv_cache.unbind(1)
# (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs))
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
torch.ops._C_cache_ops.reshape_and_cache_flash(
key,
value,
@@ -1273,7 +1275,7 @@ class FlexAttentionImpl(AttentionImpl):
key: shape = [num_tokens, num_kv_heads, head_size]
value: shape = [num_tokens, num_kv_heads, head_size]
kv_cache: shape =
[num_blocks, 2, block_size, num_kv_heads, head_size]
[num_blocks, num_kv_heads, block_size, 2 * head_size]
attn_metadata: Metadata for attention.
Returns:
shape = [num_tokens, num_heads * head_size]
@@ -1347,9 +1349,11 @@ class FlexAttentionImpl(AttentionImpl):
else:
assert self.attn_type == AttentionType.DECODER
key_cache, value_cache = kv_cache.unbind(1)
kv_cache = kv_cache.transpose(1, 2)
hs = self.head_size
key_cache, value_cache = kv_cache.split(hs, dim=-1)
# Flatten (num_blocks, block_size) into a single token dim
# Flatten (num_blocks, block_size) into a single token dim.
key_cache = key_cache.view(-1, self.num_kv_heads, self.head_size)
value_cache = value_cache.view(-1, self.num_kv_heads, self.head_size)
query, key_tensor, value_tensor = map(
+8 -4
View File
@@ -768,7 +768,8 @@ class AiterFlashAttentionBackend(AttentionBackend):
) -> tuple[int, ...]:
if block_size % 16 != 0:
raise ValueError("Block size must be a multiple of 16.")
return (num_blocks, 2, block_size, num_kv_heads, head_size)
# K and V are packed into the content dim: logical (B, H, N, 2*hs).
return (num_blocks, num_kv_heads, block_size, 2 * head_size)
@classmethod
def supports_compute_capability(cls, capability: DeviceCapability) -> bool:
@@ -1061,7 +1062,8 @@ class AiterFlashAttentionImpl(AttentionImpl):
# Whenever making a change in this method, please benchmark the
# performance to make sure it does not introduce any overhead.
num_actual_tokens = attn_metadata.num_actual_tokens
key_cache, value_cache = kv_cache.unbind(1)
# (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs))
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
if is_quantized_kv_cache(self.kv_cache_dtype):
key_cache = key_cache.view(current_platform.fp8_dtype())
@@ -1388,7 +1390,8 @@ class AiterFlashAttentionImpl(AttentionImpl):
kv_cache: torch.Tensor,
slot_mapping: torch.Tensor,
):
key_cache, value_cache = kv_cache.unbind(1)
# (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs))
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
# key and value may be None in the case of cross attention. They are
# calculated once based on the output from the encoder and then cached
@@ -1454,7 +1457,8 @@ class AiterFlashAttentionImpl(AttentionImpl):
kv_cache: torch.Tensor,
layer_slot_mapping: torch.Tensor,
):
key_cache, value_cache = kv_cache.unbind(1)
# (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs))
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
flash_layout = True
is_fp8_kv_cache = is_quantized_kv_cache(self.kv_cache_dtype)
@@ -87,7 +87,8 @@ class RocmAiterUnifiedAttentionBackend(RocmAttentionBackend):
) -> tuple[int, ...]:
if block_size % 16 != 0:
raise ValueError("Block size must be a multiple of 16.")
return (num_blocks, 2, block_size, num_kv_heads, head_size)
# K and V are packed into the content dim: logical (B, H, N, 2*hs).
return (num_blocks, num_kv_heads, block_size, 2 * head_size)
@staticmethod
def use_cascade_attention(*args, **kwargs) -> bool:
@@ -150,10 +151,8 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl):
def _split_kv_cache(
self, kv_cache: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
# Blocks-first ``(num_blocks, 2, ...)``. The model runner normalizes any
# shared decoder/cross-attention allocation to this layout, so no
# per-backend restriding is needed here.
return kv_cache.unbind(1)
# (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs))
return kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
def forward(
self,
+59 -41
View File
@@ -324,6 +324,7 @@ class TritonAttentionBackend(AttentionBackend):
) -> tuple[int, ...]:
if block_size % 16 != 0:
raise ValueError("Block size must be a multiple of 16.")
# K and V are packed into the content dim: logical (B, H, N, 2*hs).
if kv_cache_uses_per_token_head_scales(cache_dtype_str):
# Pad the head dim by sizeof(float32)/sizeof(cache_dtype) so the
# per-(token, head) scale fits inline after the quantized data;
@@ -341,26 +342,30 @@ class TritonAttentionBackend(AttentionBackend):
data_head_size = head_size // 2
else:
data_head_size = head_size
return (num_blocks, 2, block_size, num_kv_heads, data_head_size + scale_pad)
return (num_blocks, 2, block_size, num_kv_heads, head_size)
padded_hs = data_head_size + scale_pad
return (num_blocks, num_kv_heads, block_size, 2 * padded_hs)
return (num_blocks, num_kv_heads, block_size, 2 * head_size)
@staticmethod
def get_kv_cache_stride_order(
include_num_layers_dimension: bool = False,
) -> tuple[int, ...]:
# `stride_order` indicates the permutation that gets
# us from `get_kv_cache_shape` to the actual memory layout we want.
# `stride_order` indicates the permutation that gets us from
# `get_kv_cache_shape` (logical (B, H, N, 2*hs)) to the actual memory
# layout we want.
cache_layout = get_kv_cache_layout()
if cache_layout == "NHD" and include_num_layers_dimension:
# (num_blocks, num_layers, 2, block_size, num_kv_heads, head_size)
return (1, 0, 2, 3, 4, 5)
# (num_blocks, num_layers, block_size, num_kv_heads, 2*head_size)
return (1, 0, 3, 2, 4)
elif cache_layout == "NHD":
stride_order = (0, 1, 2, 3, 4)
# (num_blocks, block_size, num_kv_heads, 2*head_size)
stride_order = (0, 2, 1, 3)
elif cache_layout == "HND" and include_num_layers_dimension:
# (num_blocks, num_kv_heads, num_layers, 2, block_size, head_size)
return (1, 4, 0, 2, 3, 5)
# (num_blocks, num_kv_heads, num_layers, block_size, 2*head_size)
return (1, 2, 0, 3, 4)
elif cache_layout == "HND":
stride_order = (0, 1, 3, 2, 4)
# (num_blocks, num_kv_heads, block_size, 2*head_size)
stride_order = (0, 1, 2, 3)
else:
raise ValueError(f"Unknown cache layout: {cache_layout}")
return stride_order
@@ -410,12 +415,16 @@ class TritonAttentionImpl(AttentionImpl):
_v_scale_cache: torch.Tensor | None = None
def _ensure_scale_caches(self, kv_cache: torch.Tensor) -> None:
"""Extract per-head scale views from the padded head dimension.
"""Extract per-head scale views from the padded content dimension.
The KV cache shape is ``(num_blocks, 2, block_size, nkv, hs+pad)``
where ``pad = sizeof(float32) / sizeof(cache_dtype)``. The last
``pad`` elements of each head hold one float32 scale. We create
strided float32 views over those bytes.
The KV cache is packed as logical shape
``(num_blocks, nkv, block_size, 2 * (hs + pad))`` where
``pad = sizeof(float32) / sizeof(cache_dtype)``. The content dim holds
``[K(hs) | K_scale(pad) | V(hs) | V_scale(pad)]`` per (head, slot); the
last ``pad`` elements of each half hold one float32 scale. We create
strided float32 views over those bytes. ``kv_cache`` must be the
packed logical tensor (call before any transpose), but may have HND or
NHD physical strides.
Scale shape: ``(num_blocks, block_size, num_kv_heads)``
"""
@@ -423,9 +432,10 @@ class TritonAttentionImpl(AttentionImpl):
return
from vllm.utils.torch_utils import get_dtype_size
num_blocks, _, block_size, nkv, padded_hs = kv_cache.shape
num_blocks, nkv, block_size, content = kv_cache.shape
dtype_sz = kv_cache.element_size()
scale_pad = get_dtype_size(torch.float32) // dtype_sz # e.g. 4
padded_hs = content // 2
hs = padded_hs - scale_pad
raw = kv_cache.untyped_storage()
@@ -433,31 +443,37 @@ class TritonAttentionImpl(AttentionImpl):
raw
)
# In the raw bytes, each (block, kv_half, slot, head) occupies
# padded_hs * dtype_sz bytes. The scale float32 sits at byte
# offset hs * dtype_sz within that region.
kv_half_bytes = block_size * nkv * padded_hs * dtype_sz
full_block_f32 = 2 * kv_half_bytes // 4 # stride between blocks
slot_f32 = nkv * padded_hs * dtype_sz // 4 # stride between slots
head_f32 = padded_hs * dtype_sz // 4 # stride between heads
scale_off_f32 = hs * dtype_sz // 4 # offset to scale within head
def to_f32_units(elements: int) -> int:
nbytes = elements * dtype_sz
assert nbytes % 4 == 0
return nbytes // 4
# K scales: kv_half=0
# Actual strides (in float32 units) from the tensor. The logical cache
# may be physically NHD, so do not assume C-contiguous HND layout.
strides = kv_cache.stride()
block_f32 = to_f32_units(strides[0])
head_f32 = to_f32_units(strides[1])
slot_f32 = to_f32_units(strides[2])
# Scale sits at byte offset hs within each (K, then V) content half.
base_off_f32 = to_f32_units(kv_cache.storage_offset())
k_scale_off_f32 = base_off_f32 + to_f32_units(hs)
v_scale_off_f32 = base_off_f32 + to_f32_units(padded_hs + hs)
# K scales (first content half)
self._k_scale_cache = torch.as_strided(
base_f32,
size=(num_blocks, block_size, nkv),
stride=(full_block_f32, slot_f32, head_f32),
storage_offset=scale_off_f32,
stride=(block_f32, slot_f32, head_f32),
storage_offset=k_scale_off_f32,
)
self._k_scale_cache.fill_(1.0)
# V scales: kv_half=1, offset by kv_half_bytes
v_base_f32 = kv_half_bytes // 4
# V scales (second content half)
self._v_scale_cache = torch.as_strided(
base_f32,
size=(num_blocks, block_size, nkv),
stride=(full_block_f32, slot_f32, head_f32),
storage_offset=v_base_f32 + scale_off_f32,
stride=(block_f32, slot_f32, head_f32),
storage_offset=v_scale_off_f32,
)
self._v_scale_cache.fill_(1.0)
@@ -576,7 +592,7 @@ class TritonAttentionImpl(AttentionImpl):
key: shape = [num_tokens, num_kv_heads, head_size]
value: shape = [num_tokens, num_kv_heads, head_size]
kv_cache: shape =
[num_blocks, 2, block_size, num_kv_heads, head_size]
[num_blocks, num_kv_heads, block_size, 2 * head_size]
attn_metadata: Metadata for attention.
Returns:
shape = [num_tokens, num_heads * head_size]
@@ -617,6 +633,7 @@ class TritonAttentionImpl(AttentionImpl):
layer,
)
# KV cache arrives in logical (B, H, N, 2*hs) order.
# Per-token-head quantized KV cache: handled by the core unified
# kernel, which dequantizes per-(token, head) inline via constexpr
# branches (INT8 / FP8) and dispatches to the packed INT4 kernel.
@@ -627,7 +644,9 @@ class TritonAttentionImpl(AttentionImpl):
q_descale = k_descale = v_descale = None
# FP8 per-tensor / auto path (original flow).
else:
key_cache, value_cache = kv_cache.unbind(1)
kv_cache = kv_cache.transpose(1, 2)
hs = self.head_size
key_cache, value_cache = kv_cache.split(hs, dim=-1)
if (
is_quantized_kv_cache(self.kv_cache_dtype)
and key_cache.dtype != self.fp8_dtype
@@ -711,7 +730,8 @@ class TritonAttentionImpl(AttentionImpl):
) -> tuple[torch.Tensor, torch.Tensor]:
"""Per-token-head K/V cache views (ensures scale caches; FP8 retyped)."""
self._ensure_scale_caches(kv_cache)
key_cache, value_cache = kv_cache.unbind(1)
padded_hs = kv_cache.shape[-1] // 2
key_cache, value_cache = kv_cache.transpose(1, 2).split(padded_hs, dim=-1)
if self._kv_quant_mode == KVQuantMode.FP8_PER_TOKEN_HEAD:
key_cache = key_cache.view(self.fp8_dtype)
value_cache = value_cache.view(self.fp8_dtype)
@@ -778,13 +798,9 @@ class TritonAttentionImpl(AttentionImpl):
return
# Reshape the input keys and values and store them in the cache.
if self._is_per_token_head_quant:
self._ensure_scale_caches(kv_cache)
key_cache, value_cache = kv_cache.unbind(1)
key_cache, value_cache = self._pth_key_value_caches(kv_cache)
k_scale_cache = self._k_scale_cache
v_scale_cache = self._v_scale_cache
if self._kv_quant_mode == KVQuantMode.FP8_PER_TOKEN_HEAD:
key_cache = key_cache.view(self.fp8_dtype)
value_cache = value_cache.view(self.fp8_dtype)
triton_reshape_and_cache_flash_per_token_head_quant(
key,
value,
@@ -797,7 +813,8 @@ class TritonAttentionImpl(AttentionImpl):
)
return
# For decoder and cross-attention, use KV cache as before.
key_cache, value_cache = kv_cache.unbind(1)
# (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs))
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
if is_quantized_kv_cache(self.kv_cache_dtype):
key_cache = key_cache.view(self.fp8_dtype)
value_cache = value_cache.view(self.fp8_dtype)
@@ -829,7 +846,8 @@ class TritonAttentionImpl(AttentionImpl):
kv_cache: torch.Tensor,
layer_slot_mapping: torch.Tensor,
):
key_cache, value_cache = kv_cache.unbind(1)
# (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs))
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
flash_layout = True
is_fp8_kv_cache = is_quantized_kv_cache(self.kv_cache_dtype)
@@ -2,12 +2,9 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Triton attention backend with different K/V head dimensions (DiffKV).
The KV cache layout is identical to ``FlashAttentionDiffKVBackend`` — K
and V are packed along the last dim:
[num_blocks, block_size, num_kv_heads, head_size_qk + head_size_v]
so existing helpers (``triton_reshape_and_cache_flash_diffkv``) are reused.
The KV cache layout is identical to ``FlashAttentionDiffKVBackend``: K and V
are packed along the last dim in the logical shape
``[num_blocks, num_kv_heads, block_size, head_size_qk + head_size_v]``.
"""
from typing import ClassVar
@@ -106,10 +103,12 @@ class TritonAttentionDiffKVBackend(TritonAttentionBackend):
) -> tuple[int, ...]:
if block_size % 16 != 0:
raise ValueError("Block size must be a multiple of 16.")
# Logical (blocks-first, head-major) layout: K and V (with their
# different head sizes) packed in the content dim.
return (
num_blocks,
block_size,
num_kv_heads,
block_size,
head_size + TritonAttentionDiffKVBackend.head_size_v,
)
@@ -117,19 +116,22 @@ class TritonAttentionDiffKVBackend(TritonAttentionBackend):
def get_kv_cache_stride_order(
include_num_layers_dimension: bool = False,
) -> tuple[int, ...]:
# `stride_order` indicates the permutation that gets us from
# `get_kv_cache_shape` (logical (B, H, N, C_k+C_v)) to the actual
# memory layout we want.
cache_layout = get_kv_cache_layout()
if cache_layout == "NHD" and include_num_layers_dimension:
# (num_blocks, num_layers, block_size,
# num_kv_heads, head_size + head_size_v)
return (1, 0, 2, 3, 4)
# (num_blocks, num_layers, block_size, num_kv_heads, C_k+C_v)
return (1, 0, 3, 2, 4)
elif cache_layout == "NHD":
return (0, 1, 2, 3)
elif cache_layout == "HND" and include_num_layers_dimension:
# (num_blocks, num_kv_heads, num_layers,
# block_size, head_size + head_size_v)
return (1, 3, 0, 2, 4)
elif cache_layout == "HND":
# (num_blocks, block_size, num_kv_heads, C_k+C_v)
return (0, 2, 1, 3)
elif cache_layout == "HND" and include_num_layers_dimension:
# (num_blocks, num_kv_heads, num_layers, block_size, C_k+C_v)
return (1, 2, 0, 3, 4)
elif cache_layout == "HND":
# (num_blocks, num_kv_heads, block_size, C_k+C_v)
return (0, 1, 2, 3)
else:
raise ValueError(f"Unknown cache layout format {cache_layout}.")
@@ -175,13 +177,12 @@ class TritonAttentionDiffKVImpl(TritonAttentionImpl):
kv_cache: torch.Tensor,
slot_mapping: torch.Tensor,
) -> None:
# Cache is packed [..., head_size_qk + head_size_v]; the diffkv
# reshape kernel writes K to [..., :head_size_qk] and V to
# [..., head_size_qk:hqk+hv].
# Cache is logical (B, H, N, C); the diffkv reshape kernel expects
# (B, N, H, C).
triton_reshape_and_cache_flash_diffkv(
key,
value,
kv_cache,
kv_cache.transpose(1, 2),
slot_mapping,
self.kv_cache_dtype,
layer._k_scale,
@@ -210,7 +211,7 @@ class TritonAttentionDiffKVImpl(TritonAttentionImpl):
query: [num_tokens, num_heads, head_size_qk]
key: [num_tokens, num_kv_heads, head_size_qk]
value: [num_tokens, num_kv_heads, head_size_v]
kv_cache: [num_blocks, block_size, num_kv_heads,
kv_cache: [num_blocks, num_kv_heads, block_size,
head_size_qk + head_size_v]
output: [num_tokens, num_heads, head_size_v]
"""
@@ -231,8 +232,8 @@ class TritonAttentionDiffKVImpl(TritonAttentionImpl):
head_size_qk = self.head_size
head_size_v = TritonAttentionDiffKVBackend.head_size_v
# Slice the packed cache into K / V views. Strides on dims 0/1/2
# match the original cache; dim 3 stays contiguous (stride 1).
# Triton DiffKV kernels consume (B, N, H, D) cache views.
kv_cache = kv_cache.transpose(1, 2)
key_cache = kv_cache[..., :head_size_qk]
value_cache = kv_cache[..., head_size_qk : head_size_qk + head_size_v]
@@ -141,9 +141,10 @@ class TurboQuantAttentionBackend(AttentionBackend):
Standard attention backends use (2, num_blocks, block_size, num_kv_heads,
head_dim) with a leading 2 to separate K and V. TurboQuant packs K+V
into a single interleaved slot per head per position, so the cache is:
into a single interleaved slot per head per position. The logical
(blocks-first, head-major) shape is:
(num_blocks, block_size, num_kv_heads, slot_size_aligned)
(num_blocks, num_kv_heads, block_size, slot_size_aligned)
Each slot = [key_packed | value_packed | padding].
This is safe because TQ has its own get_kv_cache_shape override and
@@ -159,7 +160,7 @@ class TurboQuantAttentionBackend(AttentionBackend):
)
tq_config = TurboQuantConfig.from_cache_dtype(cache_dtype_str, head_size)
return (num_blocks, block_size, num_kv_heads, tq_config.slot_size_aligned)
return (num_blocks, num_kv_heads, block_size, tq_config.slot_size_aligned)
@classmethod
def supports_kv_cache_dtype(cls, kv_cache_dtype: CacheDType | None) -> bool:
@@ -422,6 +423,8 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]):
k = key[:N].view(N, self.num_kv_heads, self.head_size)
v = value[:N].view(N, self.num_kv_heads, self.head_size)
# (B, H, N, C) -> (B, N, H, C) for TQ kernels
kv_cache = kv_cache.transpose(1, 2)
self._store_kv(k, v, kv_cache, slot_mapping, layer)
def forward(
@@ -449,6 +452,9 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]):
if attn_metadata is None:
return output.fill_(0)
# (B, H, N, C) -> (B, N, H, C) for TQ kernels
kv_cache = kv_cache.transpose(1, 2)
# Slice to actual tokens
N = attn_metadata.num_actual_tokens
if N <= 0:
@@ -473,6 +473,7 @@ def reshape_and_cache_kernel_flash_diffkv(
value_stride: tl.int64,
block_stride: tl.int64,
page_stride: tl.int64,
head_stride: tl.int64,
num_heads: tl.constexpr,
head_size_k: tl.constexpr,
head_size_v: tl.constexpr,
@@ -498,9 +499,7 @@ def reshape_and_cache_kernel_flash_diffkv(
src_value_idx = token_idx * value_stride + tile_i * head_size_v
tgt_idx = (
block_idx * block_stride
+ block_offset * page_stride
+ tile_i * (head_size_k + head_size_v)
block_idx * block_stride + block_offset * page_stride + tile_i * head_stride
)
# [TILE_SIZE]
@@ -542,7 +541,7 @@ def reshape_and_cache_kernel_flash_diffkv(
def triton_reshape_and_cache_flash_diffkv(
key: torch.Tensor, # [num_tokens, num_heads, head_size]
value: torch.Tensor, # [num_tokens, num_heads, head_size_v]
# [num_blocks, block_size, num_heads, head_size + head_size_v]
# Strided [num_blocks, block_size, num_heads, head_size + head_size_v].
kv_cache: torch.Tensor,
slot_mapping: torch.Tensor, # [num_tokens]
kv_cache_dtype: str, # "auto", "fp8"
@@ -558,6 +557,7 @@ def triton_reshape_and_cache_flash_diffkv(
v_stride = value.stride()[0]
block_stride = kv_cache.stride()[0]
page_stride = kv_cache.stride()[1]
head_stride = kv_cache.stride()[2]
kv_cache_torch_dtype = (
current_platform.fp8_dtype()
@@ -599,6 +599,7 @@ def triton_reshape_and_cache_flash_diffkv(
value_stride=v_stride,
block_stride=block_stride,
page_stride=page_stride,
head_stride=head_stride,
num_heads=num_heads,
head_size_k=head_size_k,
head_size_v=head_size_v,
@@ -389,7 +389,7 @@ def triton_turboquant_store(
_tq_fused_store_fp8[grid](
k_flat,
v_flat,
kv_cache.view(-1),
kv_cache,
slot_mapping,
stride_cache_block=stride_block,
stride_cache_pos=stride_pos,
@@ -425,7 +425,7 @@ def triton_turboquant_store(
norms.squeeze(1),
v_flat,
midpoints,
kv_cache.view(-1),
kv_cache,
slot_mapping,
stride_cache_block=stride_block,
stride_cache_pos=stride_pos,