From 51878e5b6e4cf27352c427b8508b4f338ba39c9c Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Sat, 11 Jul 2026 11:11:16 -0400 Subject: [PATCH] [2/N][KV-Cache Layout Refactor] Pack K/V into the content dim across attention backends (#44455) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Lucas Wilkinson Signed-off-by: Matthew Bonanni Signed-off-by: Lucas Wilkinson Co-authored-by: Claude Co-authored-by: OpenAI Codex Co-authored-by: Matthew Bonanni Co-authored-by: Nicolò Lucchesi --- ...minimax_m3_qknorm_rope_kv_insert_kernel.cu | 57 ++++--- tests/compile/passes/test_fusion_attn.py | 9 +- tests/kernels/attention/test_cache.py | 8 +- .../test_flashinfer_trtllm_attention.py | 30 ++-- tests/kernels/attention/test_minimax_m3.py | 77 +++++----- tests/v1/attention/test_attention_backends.py | 22 ++- .../test_trtllm_attention_integration.py | 82 +++++----- .../unit/test_mooncake_connector.py | 102 +++++-------- .../kv_connector/unit/test_nixl_connector.py | 79 ++++++++-- .../unit/test_transfer_topology_sharded.py | 4 +- tests/v1/worker/test_gpu_model_runner.py | 12 +- .../kv_transfer/kv_connector/utils.py | 69 ++++----- .../kv_connector/v1/moriio/moriio_layout.py | 30 ++++ .../kv_connector/v1/nixl/base_worker.py | 124 +++++----------- .../minimax_m3/common/ops/sparse_attn.py | 35 ++--- .../minimax_m3/common/sparse_attention.py | 13 +- .../minimax_m3/nvidia/sparse_attention_msa.py | 3 +- vllm/platforms/rocm.py | 18 ++- vllm/utils/torch_utils.py | 56 ++----- vllm/v1/attention/backend.py | 12 +- vllm/v1/attention/backends/flash_attn.py | 31 ++-- .../attention/backends/flash_attn_diffkv.py | 44 +++--- vllm/v1/attention/backends/flashinfer.py | 140 ++++++++++++------ vllm/v1/attention/backends/flex_attention.py | 18 ++- vllm/v1/attention/backends/rocm_aiter_fa.py | 12 +- .../backends/rocm_aiter_unified_attn.py | 9 +- vllm/v1/attention/backends/triton_attn.py | 100 ++++++++----- .../attention/backends/triton_attn_diffkv.py | 47 +++--- vllm/v1/attention/backends/turboquant_attn.py | 12 +- .../ops/triton_reshape_and_cache_flash.py | 9 +- .../attention/ops/triton_turboquant_store.py | 4 +- 31 files changed, 664 insertions(+), 604 deletions(-) diff --git a/csrc/libtorch_stable/fused_minimax_m3_qknorm_rope_kv_insert_kernel.cu b/csrc/libtorch_stable/fused_minimax_m3_qknorm_rope_kv_insert_kernel.cu index 5429ab12d8..a2162a03b8 100644 --- a/csrc/libtorch_stable/fused_minimax_m3_qknorm_rope_kv_insert_kernel.cu +++ b/csrc/libtorch_stable/fused_minimax_m3_qknorm_rope_kv_insert_kernel.cu @@ -288,16 +288,15 @@ __global__ void fusedMiniMaxM3QNormRopeKVInsertKernel( int64_t const* __restrict__ positions, // [N] i64 int64_t const* __restrict__ slot_mapping, // main K/V slots or nullptr int64_t const* __restrict__ index_slot_mapping, // index K slots/nullptr - cache_t* __restrict__ kv_cache, // [nb,2,bs,nkv,128] or nullptr + cache_t* __restrict__ kv_cache, // [nb,nkv,bs,2*128] or nullptr out_idx_t* __restrict__ index_cache, // [nb*bs, 128]; scalar_t or e4m3 byte float const eps, int const rotary_dim, int const num_tokens, int const nq, int const nkv, int const niq, int const block_size, - // kv_cache strides (in elements) for logical shape [nb, 2, bs, nkv, 128]. - // The head_dim (last) dim is always innermost-contiguous (stride 1), so the - // NHD/HND layout choice is fully captured by these four strides: NHD keeps - // s_token < s_head, HND swaps them. dim_base addresses head_dim directly. - int64_t const kv_s_block, int64_t const kv_s_kv, int64_t const kv_s_token, - int64_t const kv_s_head) { + // kv_cache strides (in elements) for logical shape [nb, nkv, bs, 2*128]. + // The content (last) dim is always innermost-contiguous (stride 1), so the + // NHD/HND layout choice is captured by the head/token strides. + int64_t const kv_s_block, int64_t const kv_s_head, int64_t const kv_s_token, + int64_t const kv_s_dim) { #if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM) // _typeConvert is unavailable on pre-Ampere; the M3 kernel only // runs with bf16/fp16 inputs in practice. Discard the bf16 body there. @@ -438,16 +437,16 @@ __global__ void fusedMiniMaxM3QNormRopeKVInsertKernel( storeElems(index_cache + sm * kHeadDim + dim_base, elems); } } else if (isK || isV) { - // kv_cache logical shape [num_blocks, 2, block_size, nkv, head_dim]. + // kv_cache logical shape [num_blocks, nkv, block_size, 2*head_dim]. // Paging is logical (block = sm/block_size, token = sm%block_size); // the physical NHD/HND layout is honoured via the passed strides. int64_t const b = sm / block_size; int64_t const t = sm % block_size; int const kv = isK ? 0 : 1; - int64_t const off = - b * kv_s_block + kv * kv_s_kv + t * kv_s_token + head * kv_s_head; - storeCacheElems(kv_cache + off + dim_base, - elems); + int64_t const off = b * kv_s_block + head * kv_s_head + + t * kv_s_token + + (kv * kHeadDim + dim_base) * kv_s_dim; + storeCacheElems(kv_cache + off, elems); } } } @@ -474,8 +473,8 @@ void launchFusedMiniMaxM3( int64_t const* index_slot_mapping, cache_t* kv_cache, void* index_cache, float const eps, int const rotary_dim, int const num_tokens, int const nq, int const nkv, int const niq, int const block_size, - int64_t const kv_s_block, int64_t const kv_s_kv, int64_t const kv_s_token, - int64_t const kv_s_head, bool const has_index, bool const insert_kv, + int64_t const kv_s_block, int64_t const kv_s_head, int64_t const kv_s_token, + int64_t const kv_s_dim, bool const has_index, bool const insert_kv, bool const fp8_idx, cudaStream_t stream) { // Index outputs are scalar_t (bf16) or e4m3 bytes (uint8_t); reinterpret the // void* pointers per instantiation in the LAUNCH macro. @@ -517,20 +516,20 @@ void launchFusedMiniMaxM3( iq_norm_w, ik_norm_w, cos_sin_cache, positions, slot_mapping, \ index_slot_mapping, kv_cache, reinterpret_cast(index_cache), \ eps, rotary_dim, num_tokens, nq, nkv, niq, block_size, kv_s_block, \ - kv_s_kv, kv_s_token, kv_s_head) + kv_s_head, kv_s_token, kv_s_dim) #else // ROCm: standard kernel launch syntax (no PDL/stream serialization). // clang-format off #define LAUNCH(IS_SPARSE, INSERT, FP8, OUT_T) \ - fusedMiniMaxM3QNormRopeKVInsertKernel \ <<>>( \ qkv, q_out, reinterpret_cast(index_q_out), q_norm_w, \ k_norm_w, iq_norm_w, ik_norm_w, cos_sin_cache, positions, \ slot_mapping, index_slot_mapping, kv_cache, \ reinterpret_cast(index_cache), eps, rotary_dim, \ - num_tokens, nq, nkv, niq, block_size, kv_s_block, kv_s_kv, \ - kv_s_token, kv_s_head) + num_tokens, nq, nkv, niq, block_size, kv_s_block, kv_s_head, \ + kv_s_token, kv_s_dim) // clang-format on #endif @@ -584,8 +583,8 @@ void launchFusedMiniMaxM3( ? reinterpret_cast(index_cache->data_ptr()) \ : nullptr, \ static_cast(eps), static_cast(rotary_dim), num_tokens, nq, \ - nkv, niq, static_cast(block_size), kv_s_block, kv_s_kv, kv_s_token, \ - kv_s_head, has_index, insert_kv, fp8_idx, stream) + nkv, niq, static_cast(block_size), kv_s_block, kv_s_head, \ + kv_s_token, kv_s_dim, has_index, insert_kv, fp8_idx, stream) // ──────────────────────────────────────────────────────────────────────────── // Torch op wrapper @@ -602,7 +601,7 @@ void fused_minimax_m3_qknorm_rope_kv_insert( int64_t num_index_heads, // niq; 0 => dense std::optional slot_mapping, // [N] i64 std::optional index_slot_mapping, // [N] i64 - std::optional kv_cache, // [nb,2,bs,nkv,128] + std::optional kv_cache, // [nb,nkv,bs,2*128] std::optional index_cache, // [nb,bs,128] int64_t block_size, std::optional q_out, // [N, nq*128] contiguous @@ -673,11 +672,11 @@ void fused_minimax_m3_qknorm_rope_kv_insert( index_k_norm_weight->numel() == kHeadDim, "index norm weights must have 128 elements"); } - // kv_cache strides (logical shape [nb, 2, bs, nkv, head_dim]). Read straight + // kv_cache strides (logical shape [nb, nkv, bs, 2*head_dim]). Read straight // off the tensor so the kernel honours whatever physical layout the attention - // backend allocated (NHD: stride order (0,1,2,3,4); HND: (0,1,3,2,4)). No new + // backend allocated (NHD: stride order (0,2,1,3); HND: (0,1,2,3)). No new // op argument is needed -- the strides ride along with the tensor itself. - int64_t kv_s_block = 0, kv_s_kv = 0, kv_s_token = 0, kv_s_head = 0; + int64_t kv_s_block = 0, kv_s_head = 0, kv_s_token = 0, kv_s_dim = 0; torch::stable::Tensor const* effective_index_slot_mapping = nullptr; if (insert_kv) { STD_TORCH_CHECK( @@ -707,13 +706,13 @@ void fused_minimax_m3_qknorm_rope_kv_insert( index_cache->scalar_type() == torch::headeronly::ScalarType::Float8_e4m3fn), "insert mode requires index_cache matching qkv dtype or fp8 e4m3"); - STD_TORCH_CHECK(kv_cache->dim() == 5 && kv_cache->stride(4) == 1, - "kv_cache must be [nb,2,bs,nkv,head_dim] with contiguous " - "head_dim (stride(4)==1)"); + STD_TORCH_CHECK(kv_cache->dim() == 4 && kv_cache->stride(3) == 1, + "kv_cache must be [nb,nkv,bs,2*head_dim] with contiguous " + "content dim (stride(3)==1)"); kv_s_block = kv_cache->stride(0); - kv_s_kv = kv_cache->stride(1); + kv_s_head = kv_cache->stride(1); kv_s_token = kv_cache->stride(2); - kv_s_head = kv_cache->stride(3); + kv_s_dim = kv_cache->stride(3); effective_index_slot_mapping = index_slot_mapping.has_value() ? &index_slot_mapping.value() : &slot_mapping.value(); diff --git a/tests/compile/passes/test_fusion_attn.py b/tests/compile/passes/test_fusion_attn.py index b776f6af98..76536f3387 100644 --- a/tests/compile/passes/test_fusion_attn.py +++ b/tests/compile/passes/test_fusion_attn.py @@ -111,7 +111,11 @@ class AttentionQuantPatternModel(torch.nn.Module): # Fetch the attention backend and kv cache shape and stride order attn_backend = self.attn.attn_backend kv_cache_shape = attn_backend.get_kv_cache_shape( - num_blocks, self.block_size, self.num_kv_heads, self.head_size + num_blocks, + self.block_size, + self.num_kv_heads, + self.head_size, + cache_dtype_str=self.attn.kv_cache_dtype, ) try: kv_cache_stride_order = attn_backend.get_kv_cache_stride_order() @@ -125,11 +129,10 @@ class AttentionQuantPatternModel(torch.nn.Module): # Create dummy KV cache raw_tensor = torch.zeros( - 2 * num_blocks * self.block_size * self.num_kv_heads * self.head_size, + kv_cache_shape, dtype=self.attn.kv_cache_torch_dtype, device=self.device, ) - raw_tensor = raw_tensor.view(kv_cache_shape) kv_cache = raw_tensor.permute(*inv_order) self.attn.kv_cache = kv_cache diff --git a/tests/kernels/attention/test_cache.py b/tests/kernels/attention/test_cache.py index 7558da1c60..6fe355f945 100644 --- a/tests/kernels/attention/test_cache.py +++ b/tests/kernels/attention/test_cache.py @@ -10,7 +10,7 @@ from tests.kernels.utils import DEFAULT_OPCHECK_TEST_UTILS, opcheck from vllm import _custom_ops as ops from vllm.model_executor.layers.quantization.utils.quant_utils import scaled_dequantize from vllm.platforms import current_platform -from vllm.utils.torch_utils import nvfp4_kv_cache_split_views, set_random_seed +from vllm.utils.torch_utils import nvfp4_split_data_scale, set_random_seed COPYING_DIRECTION = [("cuda", "cpu"), ("cuda", "cuda"), ("cpu", "cuda")] DTYPES = [torch.bfloat16, torch.float] @@ -255,10 +255,8 @@ def test_reshape_and_cache_flash( nvfp4_key_data = None nvfp4_value_data = None if kv_cache_dtype == "nvfp4": - (nvfp4_key_data,), (key_scale_cache,) = nvfp4_kv_cache_split_views(key_cache) - (nvfp4_value_data,), (value_scale_cache,) = nvfp4_kv_cache_split_views( - value_cache - ) + nvfp4_key_data, key_scale_cache = nvfp4_split_data_scale(key_cache) + nvfp4_value_data, value_scale_cache = nvfp4_split_data_scale(value_cache) if kv_cache_dtype == "nvfp4": # Global scale = amax / 448 (per-tensor) diff --git a/tests/kernels/attention/test_flashinfer_trtllm_attention.py b/tests/kernels/attention/test_flashinfer_trtllm_attention.py index 87a12c2ff3..20c34e589f 100644 --- a/tests/kernels/attention/test_flashinfer_trtllm_attention.py +++ b/tests/kernels/attention/test_flashinfer_trtllm_attention.py @@ -13,7 +13,7 @@ from vllm.platforms import current_platform from vllm.utils.math_utils import round_up from vllm.utils.torch_utils import ( nvfp4_kv_cache_full_dim, - nvfp4_kv_cache_split_views, + nvfp4_split_data_scale, set_random_seed, ) @@ -74,17 +74,18 @@ def make_nvfp4_kv_cache( kv_scale_val, dtype=torch.float32, device=kv_bf16_hnd.device ) - # Allocate in HND physical order, permute to NHD logical order. - # hnd_order swaps dims 2↔3; it is its own inverse. + # layout: (B, 2*H, N, full_dim) + # where K heads occupy the first H heads and V heads occupy the second H heads. full_dim = nvfp4_kv_cache_full_dim(head_size) - hnd_order = (0, 1, 3, 2, 4) - kv_cache = torch.zeros( - (num_blocks, 2, num_kv_heads, block_size, full_dim), + kv_cache_hnd = torch.zeros( + (num_blocks, 2 * num_kv_heads, block_size, full_dim), dtype=torch.uint8, device=kv_bf16_hnd.device, - ).permute(*hnd_order) + ) + kv_cache_nhd = kv_cache_hnd.permute(0, 2, 1, 3) + k_view_nhd, v_view_nhd = kv_cache_nhd.split(num_kv_heads, dim=-2) - # Flatten NHD [N, T, H, D] → token tensors [N*T, H, D] for the kernel. + # Flatten input KV → token tensors [B*N, H, head_size] for the kernel. num_tokens = num_blocks * block_size k_tokens = ( kv_bf16_hnd[:, 0] @@ -98,22 +99,21 @@ def make_nvfp4_kv_cache( ) slot_mapping = torch.arange(num_tokens, dtype=torch.long, device=kv_bf16_hnd.device) - # reshape_and_cache_flash: kernel receives kv_cache[:, 0] and [:, 1] - # (full K/V buffers containing both data and scale). torch.ops._C_cache_ops.reshape_and_cache_flash( k_tokens, v_tokens, - kv_cache[:, 0], - kv_cache[:, 1], + k_view_nhd, + v_view_nhd, slot_mapping, "nvfp4", kv_scale_tensor, kv_scale_tensor, ) - # Split in HND order for trtllm kernel (expects HND numTokensPerPage). - kv_cache_hnd = kv_cache.permute(*hnd_order) - (k_data, v_data), (k_scales, v_scales) = nvfp4_kv_cache_split_views(kv_cache_hnd) + # Split into data/scale views in HNC order for trtllm kernel. + k_cache_hnc, v_cache_hnc = kv_cache_hnd.split(num_kv_heads, dim=1) + k_data, k_scales = nvfp4_split_data_scale(k_cache_hnc) + v_data, v_scales = nvfp4_split_data_scale(v_cache_hnc) # Dequantize for the FA2 reference baseline. ref_k = dequant_nvfp4_kv_cache( diff --git a/tests/kernels/attention/test_minimax_m3.py b/tests/kernels/attention/test_minimax_m3.py index 01405a0e5d..6251f423f1 100644 --- a/tests/kernels/attention/test_minimax_m3.py +++ b/tests/kernels/attention/test_minimax_m3.py @@ -662,8 +662,9 @@ def _reference_sparse_attn( positions = torch.arange(seq_len, device="cuda") pages = block_table[req_id, positions // BLOCK_SIZE] rows = positions % BLOCK_SIZE - k_req = kv_cache[pages, 0, rows] - v_req = kv_cache[pages, 1, rows].float() + kv_req = kv_cache[pages, :, rows] + k_req = kv_req[..., :HEAD_DIM] + v_req = kv_req[..., HEAD_DIM:].float() q_pos = prefix_len + torch.arange(q_len, device="cuda") key_blocks = positions // BLOCK_SIZE @@ -791,15 +792,15 @@ def test_main_backend_layout_contract(): flash_attn-style stride order for each layout.""" nb, bs, h, d = 7, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM logical = MiniMaxM3SparseBackend.get_kv_cache_shape(nb, bs, h, d) - assert logical == (nb, 2, bs, h, d) - # The old HND-ordered shape is no longer the logical shape. - assert logical != (nb, 2, h, bs, d) + assert logical == (nb, h, bs, 2 * d) + # The old separate K/V-axis shape is no longer the logical shape. + assert logical != (nb, 2, bs, h, d) try: set_kv_cache_layout("HND") - assert MiniMaxM3SparseBackend.get_kv_cache_stride_order() == (0, 1, 3, 2, 4) + assert MiniMaxM3SparseBackend.get_kv_cache_stride_order() == (0, 1, 2, 3) set_kv_cache_layout("NHD") - assert MiniMaxM3SparseBackend.get_kv_cache_stride_order() == (0, 1, 2, 3, 4) + assert MiniMaxM3SparseBackend.get_kv_cache_stride_order() == (0, 2, 1, 3) finally: set_kv_cache_layout(None) @@ -829,7 +830,7 @@ def test_main_backend_unknown_layout_raises(monkeypatch): def test_indexer_backend_stride_order_is_identity(): - """The 3-dim indexer cache must not inherit the parent's 5-element stride + """The 3-dim indexer cache must not inherit the parent's 4-element stride order; it overrides to the 3-element identity so the allocator keeps the contiguous layout.""" assert MiniMaxM3IndexerBackend.get_kv_cache_stride_order() == (0, 1, 2) @@ -848,9 +849,9 @@ def test_indexer_backend_stride_order_is_identity(): assert _stride_order_for(MiniMaxM3IndexerBackend, len(indexer_shape)) == (0, 1, 2) -def test_hnd_allocation_is_byte_identical_to_transpose(): - """Under HND the backend-visible logical view is byte-identical to the - pre-change allocate-HND-then-transpose(2, 3) workaround.""" +def test_hnd_allocation_is_packed_head_major(): + """Under HND the backend-visible logical view is the packed head-major + physical allocation.""" nb, bs, h, d = 4, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM logical = MiniMaxM3SparseBackend.get_kv_cache_shape(nb, bs, h, d) try: @@ -860,21 +861,22 @@ def test_hnd_allocation_is_byte_identical_to_transpose(): set_kv_cache_layout(None) physical_shape = tuple(logical[i] for i in stride_order) - # The physical (permuted) shape equals the old hardcoded HND shape. - assert physical_shape == (nb, 2, h, bs, d) + assert physical_shape == (nb, h, bs, 2 * d) inv_order = [stride_order.index(i) for i in range(len(stride_order))] raw = torch.empty(physical_shape, device="cuda", dtype=DTYPE) view = raw.permute(*inv_order) - expected = raw.view((nb, 2, h, bs, d)).transpose(2, 3) + expected = raw.view((nb, h, bs, 2 * d)) assert view.shape == expected.shape assert view.stride() == expected.stride() assert view.storage_offset() == expected.storage_offset() - # Negative: the identity (wrong) stride order under HND does not reproduce - # the transpose view. - wrong_view = raw.view(logical) + # Negative: the NHD stride order under HND does not reproduce the + # head-major view. + wrong_order = (0, 2, 1, 3) + wrong_inv = [wrong_order.index(i) for i in range(len(wrong_order))] + wrong_view = raw.view(tuple(logical[i] for i in wrong_order)).permute(*wrong_inv) assert wrong_view.stride() != expected.stride() @@ -1037,14 +1039,18 @@ def test_decode_wrong_layout_breaks_parity(): seq_lens_list = (130, 257) q, block_table, seq_lens, topk_idx, num_pages = _build_decode_inputs(seq_lens_list) - # Physical HND storage [blocks, 2, heads, block, dim]. + # Physical HND storage [blocks, heads, block, packed_kv_dim]. phys = torch.randn( - (num_pages, 2, NUM_KV_HEADS, BLOCK_SIZE, HEAD_DIM), device="cuda", dtype=DTYPE + (num_pages, NUM_KV_HEADS, BLOCK_SIZE, 2 * HEAD_DIM), + device="cuda", + dtype=DTYPE, + ) + # Correct logical packed-HND view vs. the same bytes mislabeled as NHD + # physical storage and then exposed as a logical cache. + correct = phys + wrong = phys.reshape(num_pages, BLOCK_SIZE, NUM_KV_HEADS, 2 * HEAD_DIM).permute( + 0, 2, 1, 3 ) - # Correct logical-NHD view (strided) vs. the same bytes mislabeled as a - # contiguous-NHD cache — same shape, different content mapping. - correct = phys.permute(0, 1, 3, 2, 4) - wrong = phys.reshape(num_pages, 2, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM) q_lens_t = torch.ones(len(seq_lens_list), device="cuda", dtype=torch.int32) prefix_lens = seq_lens - q_lens_t @@ -1072,9 +1078,9 @@ def _make_attn_group(backend, spec): def test_main_cache_byte_identical_through_production_allocator(): """AC-2: drive the real allocator (`_reshape_kv_cache`) for the M3 main `FullAttentionSpec` under HND and assert the backend-visible view has the - same shape, stride, and storage offset as the pre-change - allocate-HND-then-transpose path; the indexer `MLAAttentionSpec` allocates - through the same path to its 3-dim shape.""" + same shape, stride, and storage offset as the packed-HND allocation; the + indexer `MLAAttentionSpec` allocates through the same path to its 3-dim + shape.""" nb = 4 spec = FullAttentionSpec( block_size=BLOCK_SIZE, @@ -1092,8 +1098,7 @@ def test_main_cache_byte_identical_through_production_allocator(): set_kv_cache_layout(None) view = kv_caches["main"] - oracle = raw.view(DTYPE).view((nb, 2, NUM_KV_HEADS, BLOCK_SIZE, HEAD_DIM)) - oracle = oracle.transpose(2, 3) + oracle = raw.view(DTYPE).view((nb, NUM_KV_HEADS, BLOCK_SIZE, 2 * HEAD_DIM)) assert tuple(view.shape) == tuple(oracle.shape) assert view.stride() == oracle.stride() assert view.storage_offset() == oracle.storage_offset() @@ -1119,13 +1124,13 @@ def test_main_cache_byte_identical_through_production_allocator(): def test_indexer_inherited_stride_order_trips_allocator_assert(): - """AC-4 negative: without the indexer override, the inherited 5-element + """AC-4 negative: without the indexer override, the inherited 4-element stride order trips the allocator's `len(stride_order) == len(shape)` assert for the 3-dim indexer shape; the `AssertionError` is NOT swallowed by the allocator's `(AttributeError, NotImplementedError)` fallback.""" class _BrokenIndexerBackend(MiniMaxM3IndexerBackend): - # Simulate inheriting the parent's 5-element stride order. + # Simulate inheriting the parent's 4-element stride order. get_kv_cache_stride_order = staticmethod( MiniMaxM3SparseBackend.get_kv_cache_stride_order ) @@ -1192,10 +1197,8 @@ def test_padded_main_cache_is_flagged(): @pytest.mark.parametrize("kv_layout", ["NHD", "HND"], indirect=True) def test_reshape_and_cache_flash_write_persists(kv_layout: str): """AC-5 write path: the `reshape_and_cache_flash` write site now consumes - `self.kv_cache.unbind(1)` directly. Writing through those views must persist - into the bound storage (read back through an independent logical view) under - both layouts — a `.contiguous()` copy of the unbind slice would leave the - bound storage unchanged.""" + packed-content K/V split views. Writing through those views must persist + into the bound storage under both layouts.""" torch.manual_seed(0) num_pages = 4 kv_cache = _allocate_main_kv_via_contract(num_pages) @@ -1203,7 +1206,7 @@ def test_reshape_and_cache_flash_write_persists(kv_layout: str): kv_cache.zero_() # Exactly the production write-site code under test. - key_cache, value_cache = kv_cache.unbind(1) + key_cache, value_cache = kv_cache.transpose(1, 2).split(HEAD_DIM, dim=-1) num_tokens = 12 slot_mapping = torch.randperm(num_pages * BLOCK_SIZE, device="cuda")[ @@ -1222,5 +1225,5 @@ def test_reshape_and_cache_flash_write_persists(kv_layout: str): for t in range(num_tokens): slot = int(slot_mapping[t].item()) blk, intra = divmod(slot, BLOCK_SIZE) - torch.testing.assert_close(kv_cache[blk, 0, intra], key[t]) - torch.testing.assert_close(kv_cache[blk, 1, intra], value[t]) + torch.testing.assert_close(kv_cache[blk, :, intra, :HEAD_DIM], key[t]) + torch.testing.assert_close(kv_cache[blk, :, intra, HEAD_DIM:], value[t]) diff --git a/tests/v1/attention/test_attention_backends.py b/tests/v1/attention/test_attention_backends.py index d630037e9b..abbdeac1ef 100644 --- a/tests/v1/attention/test_attention_backends.py +++ b/tests/v1/attention/test_attention_backends.py @@ -140,12 +140,10 @@ def create_and_prepopulate_kv_cache( block_table = common_attn_metadata.block_table_tensor slot_mapping = common_attn_metadata.slot_mapping - # Create KV cache and populate in (2, num_blocks, ...) layout for easy - # flat indexing, then transpose to (num_blocks, 2, ...) layout. kv_cache = torch.zeros( - 2, num_blocks, block_size, num_kv_heads, head_size, dtype=dtype, device=device + num_blocks, block_size, num_kv_heads, 2 * head_size, dtype=dtype, device=device ) - kv_cache_flat = kv_cache.view(2, -1, num_kv_heads, head_size) + kv_cache_flat = kv_cache.view(-1, num_kv_heads, 2 * head_size) # Populate the cache with the context tokens # Start from block_id=1 since block_id=0 is considered the null block @@ -154,15 +152,12 @@ def create_and_prepopulate_kv_cache( k_context, v_context = k_contexts[i], v_contexts[i] start = start_block_idx * block_size end = start + k_context.shape[0] - kv_cache_flat[0, start:end, ...] = k_context - kv_cache_flat[1, start:end, ...] = v_context + kv_cache_flat[start:end, :, :head_size] = k_context + kv_cache_flat[start:end, :, head_size:] = v_context # Stay block aligned and allocate enough blocks for the new tokens start_block_idx += cdiv(int(seq_lens[i]), block_size) - # Transpose to (num_blocks, 2, ...) layout - kv_cache = kv_cache.transpose(0, 1).contiguous() - blocks_end = start_block_idx # Permute the context blocks (excluding block 0 which is null) @@ -199,7 +194,8 @@ def create_and_prepopulate_kv_cache( i, block_indices ] * block_size + token_inter_block_offsets.to(device) - return kv_cache + # Transpose to logical (num_blocks, num_kv_heads, block_size, 2*hs) + return kv_cache.transpose(1, 2).contiguous() class MockAttentionLayer: @@ -496,9 +492,6 @@ def _test_backend_correctness( set_kv_cache_layout("HND") reset_kv_cache_layout = True - # Apply stride order like runtime does in - # _reshape_kv_cache (attn_utils.py:182-210): permute to physical - # layout, make contiguous, then permute to logical layout. kv_cache_for_backend = kv_cache if backend_cls is not None: try: @@ -506,6 +499,9 @@ def _test_backend_correctness( except (AttributeError, NotImplementedError): stride_order = tuple(range(kv_cache.ndim)) if stride_order != tuple(range(kv_cache.ndim)): + # Apply stride order like runtime does in + # _reshape_kv_cache (attn_utils.py:182-210): permute to physical + # layout, make contiguous, then permute to logical layout. inv_order = [stride_order.index(i) for i in range(len(stride_order))] kv_cache_for_backend = ( kv_cache.permute(*stride_order).contiguous().permute(*inv_order) diff --git a/tests/v1/attention/test_trtllm_attention_integration.py b/tests/v1/attention/test_trtllm_attention_integration.py index 0fe9a7ecf6..c8a080e6d1 100644 --- a/tests/v1/attention/test_trtllm_attention_integration.py +++ b/tests/v1/attention/test_trtllm_attention_integration.py @@ -97,13 +97,13 @@ def _create_hnd_kv_cache( device, num_blocks, common_attn_metadata, + kv_in_head_dim=False, ): - """Create and populate a KV cache with HND-compatible strides. + """Create and populate a packed KV cache with HND-compatible strides. - The returned tensor has logical shape - (num_blocks, 2, block_size, num_kv_heads, head_size) but is physically - laid out as (num_blocks, 2, num_kv_heads, block_size, head_size) so that - ``kv_cache.permute(0, 1, 3, 2, 4)`` yields a contiguous HND view. + When kv_in_head_dim=False (default), returns (B, H, N, 2*hs) with K/V + packed in the content dim. When kv_in_head_dim=True, returns + (B, 2*H, N, hs) with K/V as separate head groups. """ seq_lens = common_attn_metadata.seq_lens.cpu() query_lens = ( @@ -114,26 +114,34 @@ def _create_hnd_kv_cache( slot_mapping = common_attn_metadata.slot_mapping batch_size = len(k_contexts) - # Build cache in (2, num_blocks, block_size, num_kv_heads, head_size) - # then convert to HND format (same approach as test_attention_backends.py). - kv_cache_raw = torch.zeros( - 2, + # kv_in_head_dim: (B, N, 2*H, hs) — K/V as separate head groups + # else: (B, N, H, 2*hs) — K/V packed in content dim + n_heads, content = ( + (2 * num_kv_heads, head_size) + if kv_in_head_dim + else (num_kv_heads, 2 * head_size) + ) + kv_cache = torch.zeros( num_blocks, block_size, - num_kv_heads, - head_size, + n_heads, + content, dtype=dtype, device=device, ) - kv_cache_flat = kv_cache_raw.view(2, -1, num_kv_heads, head_size) + kv_cache_flat = kv_cache.view(-1, n_heads, content) start_block_idx = 1 for i in range(batch_size): k_ctx, v_ctx = k_contexts[i], v_contexts[i] start = start_block_idx * block_size end = start + k_ctx.shape[0] - kv_cache_flat[0, start:end] = k_ctx - kv_cache_flat[1, start:end] = v_ctx + if kv_in_head_dim: + kv_cache_flat[start:end, :num_kv_heads] = k_ctx + kv_cache_flat[start:end, num_kv_heads:] = v_ctx + else: + kv_cache_flat[start:end, :, :head_size] = k_ctx + kv_cache_flat[start:end, :, head_size:] = v_ctx start_block_idx += cdiv(int(seq_lens[i]), block_size) blocks_end = start_block_idx @@ -142,7 +150,7 @@ def _create_hnd_kv_cache( perm = torch.randperm(blocks_end - 1) + 1 inv_perm = torch.zeros(blocks_end, dtype=torch.long, device=device) inv_perm[1:] = torch.argsort(perm) + 1 - kv_cache_raw[:, 1:blocks_end] = kv_cache_raw[:, perm] + kv_cache[1:blocks_end] = kv_cache[perm] # Build block table. start_block_idx = 1 @@ -165,10 +173,8 @@ def _create_hnd_kv_cache( i, block_indices ] * block_size + intra_block_offsets.to(device) - # Transpose to FlashInfer logical shape then make HND-strided. - kv_cache = kv_cache_raw.transpose(0, 1) - kv_cache = kv_cache.transpose(2, 3).contiguous().transpose(2, 3) - return kv_cache + # Transpose to canonical: (B, H, N, 2*hs) or (B, 2*H, N, hs) + return kv_cache.transpose(1, 2).contiguous() def _create_nvfp4_hnd_kv_cache( @@ -187,20 +193,15 @@ def _create_nvfp4_hnd_kv_cache( reshape_and_cache_flash, using the same block-table layout as _create_hnd_kv_cache. - The returned tensor is dtype ``uint8`` with shape - ``(num_blocks, 2, block_size, num_kv_heads, full_dim)`` in logical - (NHD) order, but physically permuted to HND layout via stride order - ``(0, 1, 3, 2, 4)`` (i.e. ``num_kv_heads`` before ``block_size``). - - The last dimension ``full_dim = head_size // 2 + head_size // 16`` - packs two regions contiguously: + The returned tensor is dtype ``uint8`` with head-group layout + ``(num_blocks, 2 * num_kv_heads, block_size, full_dim)`` + where K heads occupy the first ``num_kv_heads`` heads and V heads the second. + Each ``full_dim = head_size // 2 + head_size // 16`` block packs two regions: - **FP4 data** (``head_size // 2`` bytes): pairs of E2M1 values, two per byte. - **FP8 block scales** (``head_size // 16`` bytes): one E4M3 scale per 16-element block. - Dimension 1 indexes K (``[:, 0]``) and V (``[:, 1]``). - Args: k_contexts: List of key context tensors, one per sequence. v_contexts: List of value context tensors, one per sequence. @@ -219,6 +220,7 @@ def _create_nvfp4_hnd_kv_cache( ``torch.Tensor``: The nvfp4 kv_cache tensor (uint8, HND-strided). """ # First create a bf16 HND cache so block tables are populated. + # Use kv_in_head_dim=True so K/V are separate head groups (B, 2*H, N, hs). bf16_cache = _create_hnd_kv_cache( k_contexts, v_contexts, @@ -229,20 +231,20 @@ def _create_nvfp4_hnd_kv_cache( device, num_blocks, common_attn_metadata, + kv_in_head_dim=True, ) - # Allocate nvfp4 cache: same shape but with full_dim (data + scale). + # (num_blocks, 2 * num_kv_heads, block_size, full_dim) — K heads first, then V heads full_dim = nvfp4_kv_cache_full_dim(head_size) - hnd_order = (0, 1, 3, 2, 4) nvfp4_cache = torch.zeros( - (num_blocks, 2, num_kv_heads, block_size, full_dim), + (num_blocks, 2 * num_kv_heads, block_size, full_dim), dtype=torch.uint8, device=device, - ).permute(*hnd_order) + ) + k_cache, v_cache = nvfp4_cache.split(num_kv_heads, dim=1) # Flatten bf16 context into tokens and quantize via reshape_and_cache_flash. - # bf16_cache is (num_blocks, 2, block_size, num_kv_heads, head_size) logical - # with HND physical strides. + # bf16_cache is (B, 2*H, N, hs); split K/V on head dim. block_table = common_attn_metadata.block_table_tensor seq_lens = common_attn_metadata.seq_lens.cpu() query_lens = ( @@ -258,19 +260,21 @@ def _create_nvfp4_hnd_kv_cache( # Gather context tokens from the bf16 cache using block table. n_ctx_blocks = (ctx_len + block_size - 1) // block_size blocks = block_table[i, :n_ctx_blocks] - # bf16_cache[:, kv_idx] is (num_blocks, block_size, num_kv_heads, head_size) - k_ctx = bf16_cache[blocks, 0].reshape(-1, num_kv_heads, head_size)[:ctx_len] - v_ctx = bf16_cache[blocks, 1].reshape(-1, num_kv_heads, head_size)[:ctx_len] + # bf16_cache is (B, 2*H, N, hs); split K and V head groups. + k_bf16, v_bf16 = bf16_cache[blocks].split(num_kv_heads, dim=1) + k_ctx = k_bf16.transpose(1, 2).reshape(-1, num_kv_heads, head_size)[:ctx_len] + v_ctx = v_bf16.transpose(1, 2).reshape(-1, num_kv_heads, head_size)[:ctx_len] # Build slot mapping for these context tokens. token_offsets = torch.arange(ctx_len, device=device) block_indices = token_offsets // block_size intra_offsets = token_offsets % block_size slots = block_table[i, block_indices] * block_size + intra_offsets + # reshape_and_cache_flash expects (B, N, H, D) cache views. torch.ops._C_cache_ops.reshape_and_cache_flash( k_ctx, v_ctx, - nvfp4_cache[:, 0], - nvfp4_cache[:, 1], + k_cache.transpose(1, 2), + v_cache.transpose(1, 2), slots, "nvfp4", kv_scale_t, diff --git a/tests/v1/kv_connector/unit/test_mooncake_connector.py b/tests/v1/kv_connector/unit/test_mooncake_connector.py index a227af909e..4847b956b1 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_connector.py +++ b/tests/v1/kv_connector/unit/test_mooncake_connector.py @@ -289,10 +289,9 @@ async def test_send_kv_to_decode_aligns_consumer_regions_by_layer_metadata( prefill_worker = prefill_connector.connector_worker block_len = 4096 - kv_half = block_len // 2 prefill_worker.kv_caches_base_addr = [0x1000] prefill_worker.block_len_per_layer = [block_len] - prefill_worker.kv_block_len_per_layer = [kv_half] + prefill_worker.kv_block_len_per_layer = [block_len] prefill_worker.registered_layer_names = ["model.layers.1.self_attn"] prefill_worker.registered_layer_indices = [1] @@ -321,7 +320,7 @@ async def test_send_kv_to_decode_aligns_consumer_regions_by_layer_metadata( req_blocks={"d-req-layer-align": (transfer_id, [[20]])}, kv_caches_base_addr=[0xA000, 0xB000], block_lens=[block_len, block_len], - kv_block_lens=[kv_half, kv_half], + kv_block_lens=[block_len, block_len], registered_layer_names=[ "model.layers.0.self_attn", "model.layers.1.self_attn", @@ -338,15 +337,9 @@ async def test_send_kv_to_decode_aligns_consumer_regions_by_layer_metadata( await prefill_worker.send_kv_to_decode(identity, mock_socket, xfer_meta) src_ptrs, dst_ptrs, lengths = mock_send_blocks.call_args[0][1:] - assert src_ptrs == [ - 0x1000 + 10 * block_len, - 0x1000 + 10 * block_len + kv_half, - ] - assert dst_ptrs == [ - 0xB000 + 20 * block_len, - 0xB000 + 20 * block_len + kv_half, - ] - assert lengths == [kv_half, kv_half] + assert src_ptrs == [0x1000 + 10 * block_len] + assert dst_ptrs == [0xB000 + 20 * block_len] + assert lengths == [block_len] sent_identity, sent_payload = mock_socket.send_multipart.call_args[0][0] assert sent_identity == identity @@ -832,9 +825,8 @@ async def test_kv_producer(monkeypatch): prefill_worker = prefill_connector.connector_worker prefill_worker.kv_caches_base_addr = [0x1000] block_len = 4096 - kv_half = block_len // 2 prefill_worker.block_len_per_layer = [block_len] - prefill_worker.kv_block_len_per_layer = [kv_half] + prefill_worker.kv_block_len_per_layer = [block_len] prefill_worker.registered_layer_names = ["model.layers.0.self_attn"] prefill_worker.registered_layer_indices = [0] @@ -862,7 +854,7 @@ async def test_kv_producer(monkeypatch): req_blocks={"d-req-1": (transfer_id, [[20, 21]])}, kv_caches_base_addr=[0x2000], block_lens=[block_len], - kv_block_lens=[kv_half], + kv_block_lens=[block_len], registered_layer_names=["model.layers.0.self_attn"], registered_layer_indices=[0], ) @@ -874,24 +866,18 @@ async def test_kv_producer(monkeypatch): with patch.object( prefill_worker, "_send_blocks", return_value=0 ) as mock_send_blocks: - # With blocks-first layout, each block is virtually split - # into K and V halves, producing non-coalesced transfers. - def expected_split_transfers(src_base, dst_base, src_blocks, dst_blocks): - """Build expected (src_ptrs, dst_ptrs, lengths) for - virtual-split K/V transfers.""" - src_ptrs, dst_ptrs, lengths = [], [], [] - for kv_offset in (0, kv_half): - for sb, db in zip(src_blocks, dst_blocks): - src_ptrs.append(src_base + sb * block_len + kv_offset) - dst_ptrs.append(dst_base + db * block_len + kv_offset) - lengths.append(kv_half) - return src_ptrs, dst_ptrs, lengths + + def expected_transfers(src_base, dst_base, src_blocks, dst_blocks): + n = len(src_blocks) + return ( + [src_base + src_blocks[0] * block_len], + [dst_base + dst_blocks[0] * block_len], + [n * block_len], + ) # Normal case: 2 blocks to 2 blocks await prefill_worker.send_kv_to_decode(identity, mock_socket, xfer_meta) - src, dst, lens = expected_split_transfers( - 0x1000, 0x2000, [10, 11], [20, 21] - ) + src, dst, lens = expected_transfers(0x1000, 0x2000, [10, 11], [20, 21]) mock_send_blocks.assert_called_once_with( "consumer-host:54321", src, @@ -923,7 +909,7 @@ async def test_kv_producer(monkeypatch): # Worker processes the consumer's request await prefill_worker.send_kv_to_decode(identity, mock_socket, xfer_meta) # Verify transfer parameters are correct: 11 to 20 - src, dst, lens = expected_split_transfers(0x1000, 0x2000, [11], [20]) + src, dst, lens = expected_transfers(0x1000, 0x2000, [11], [20]) mock_send_blocks.assert_called_once_with( "consumer-host:54321", src, @@ -1266,7 +1252,7 @@ async def test_kv_producer_heterogeneous_tp(monkeypatch, d_tp_size): prefill_worker.kv_caches_base_addr = [0x1000] prefill_worker.block_len_per_layer = [local_block_len] - prefill_worker.kv_block_len_per_layer = [local_block_len // 2] + prefill_worker.kv_block_len_per_layer = [local_block_len] prefill_worker.registered_layer_names = ["model.layers.0.self_attn"] prefill_worker.registered_layer_indices = [0] @@ -1314,7 +1300,7 @@ async def test_kv_producer_heterogeneous_tp(monkeypatch, d_tp_size): }, kv_caches_base_addr=[0x2000], block_lens=[remote_block_len], - kv_block_lens=[remote_block_len // 2], + kv_block_lens=[remote_block_len], registered_layer_names=["model.layers.0.self_attn"], registered_layer_indices=[0], ) @@ -1336,47 +1322,31 @@ async def test_kv_producer_heterogeneous_tp(monkeypatch, d_tp_size): flat_remote = [b for g in remote_block_ids for b in g] num_blocks = len(flat_local) - # With blocks-first layout, virtual split halves block - # lengths and doubles transfer regions (K + V). - local_kv_block_len = local_block_len // 2 - remote_kv_block_len = remote_block_len // 2 + assert len(src_ptrs) == num_blocks + assert len(dst_ptrs) == num_blocks + assert len(lengths) == num_blocks - assert len(src_ptrs) == 2 * num_blocks - assert len(dst_ptrs) == 2 * num_blocks - assert len(lengths) == 2 * num_blocks - - # Compute expected offsets using kv_block_len if d_tp_size <= P_TP_SIZE: tp_ratio = P_TP_SIZE // d_tp_size expected_src_off = 0 - expected_dst_off = (P_TP_RANK % tp_ratio) * local_kv_block_len - expected_xfer_len = local_kv_block_len + expected_dst_off = (P_TP_RANK % tp_ratio) * local_block_len + expected_xfer_len = local_block_len else: ratio_abs = d_tp_size // P_TP_SIZE - expected_src_off = (d_rank % ratio_abs) * remote_kv_block_len + expected_src_off = (d_rank % ratio_abs) * remote_block_len expected_dst_off = 0 - expected_xfer_len = remote_kv_block_len + expected_xfer_len = remote_block_len - # First num_blocks entries are K region, - # next num_blocks are V region. - for region_idx in range(2): - local_region_base = 0x1000 + region_idx * local_kv_block_len - remote_region_base = 0x2000 + region_idx * remote_kv_block_len - for blk_idx, (lblk, rblk) in enumerate( - zip(flat_local, flat_remote) - ): - idx = region_idx * num_blocks + blk_idx - assert src_ptrs[idx] == ( - local_region_base - + lblk * local_block_len - + expected_src_off - ) - assert dst_ptrs[idx] == ( - remote_region_base - + rblk * remote_block_len - + expected_dst_off - ) - assert lengths[idx] == expected_xfer_len + local_region_base = 0x1000 + remote_region_base = 0x2000 + for blk_idx, (lblk, rblk) in enumerate(zip(flat_local, flat_remote)): + assert src_ptrs[blk_idx] == ( + local_region_base + lblk * local_block_len + expected_src_off + ) + assert dst_ptrs[blk_idx] == ( + remote_region_base + rblk * remote_block_len + expected_dst_off + ) + assert lengths[blk_idx] == expected_xfer_len # Verify successful response sent back to consumer mock_socket.send_multipart.assert_called_once() diff --git a/tests/v1/kv_connector/unit/test_nixl_connector.py b/tests/v1/kv_connector/unit/test_nixl_connector.py index 49d75e193f..0724e77693 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector.py @@ -1068,6 +1068,65 @@ class TestNixlHandshake: # whole block is moved. worker.add_remote_agent(meta, remote_tp_size=1) + @patch( + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", + FakeNixlWrapper, + ) + def test_hybrid_mamba_attention_remote_descs_use_packed_head_slices( + self, default_vllm_config, dist_init + ): + worker = FakeNixlConnectorWorker( + create_vllm_config(), "engine", hand_shake_latency=0 + ) + + remote_block_len = 2048 + local_block_len = remote_block_len // 2 + worker.block_len_per_layer = [local_block_len] + worker._region_is_mla = [False] + worker.num_blocks = 1 + worker.num_regions = 1 + worker._has_mamba = True + worker._mamba_ssm_size = (128, 256) + worker.transfer_topo = TransferTopology( + tp_rank=1, + tp_size=2, + block_size=worker.block_size, + engine_id=worker.engine_id, + is_mla=False, + is_mamba=True, + total_num_kv_heads=2, + attn_backends=worker.attn_backends, + tensor_shape=None, + ) + assert worker.transfer_topo.virtually_split_kv_in_blocks + + plan = MagicMock( + source_ranks_per_group=((0,), (0,)), + rank_offset_factor=1, + ) + meta = MagicMock( + kv_caches_base_addr=[0x1000], + device_id=0, + num_blocks=1, + block_lens=[remote_block_len], + ) + + assert worker.get_backend_aware_kv_block_len(0, mamba_view=False) == ( + local_block_len + ) + assert ( + worker.get_backend_aware_kv_block_len(0, first_split=True, mamba_view=True) + == worker._mamba_ssm_size[0] + ) + assert ( + worker.get_backend_aware_kv_block_len(0, first_split=False, mamba_view=True) + == worker._mamba_ssm_size[1] + ) + + assert worker._build_fa_remote(plan, meta, block_size_ratio=1) == [ + (0x1000 + local_block_len, local_block_len, 0) + ] + @patch( "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FakeNixlWrapper, @@ -1691,13 +1750,6 @@ def _run_abort_timeout_test(llm: LLM, timeout: int): reason="Attention backend FLASH_ATTN is not supported on ROCm", ), ), - pytest.param( - "ROCM_ATTN", - marks=pytest.mark.skipif( - not current_platform.is_rocm(), - reason="Attention backend ROCM_ATTN is only supported on ROCm", - ), - ), "TRITON_ATTN", ], ) @@ -1824,8 +1876,7 @@ def test_register_kv_caches( test_shape = backend_cls.get_kv_cache_shape( num_blocks=1, block_size=16, num_kv_heads=1, head_size=1 ) - is_blocks_first = len(test_shape) == 5 and test_shape[0] == 1 - virtually_split = is_blocks_first and not connector.prefer_cross_layer_blocks + is_blocks_first = len(test_shape) == 4 and test_shape[0] == 1 if connector.prefer_cross_layer_blocks: with set_current_vllm_config(vllm_config): @@ -1856,7 +1907,7 @@ def test_register_kv_caches( ] expected_num_entries = 1 - expected_blocks_count = num_blocks * (2 if virtually_split else 1) + expected_blocks_count = num_blocks kv_caches = {"all-layers": cross_layers_kv_cache} else: @@ -1885,6 +1936,7 @@ def test_register_kv_caches( unique_tensor.data_ptr(), ] expected_num_entries = 2 + expected_blocks_count = kv_cache_config.num_blocks * 2 else: expected_tensor_size = ( shared_tensor[0].element_size() * shared_tensor[0].numel() @@ -1896,7 +1948,7 @@ def test_register_kv_caches( unique_tensor[1].data_ptr(), ] expected_num_entries = 4 - expected_blocks_count = kv_cache_config.num_blocks * 4 + expected_blocks_count = kv_cache_config.num_blocks * 4 # Execute register_kv_caches connector.register_kv_caches(kv_caches) @@ -1930,10 +1982,7 @@ def test_register_kv_caches( else: num_blocks = kv_cache_config.num_blocks - if virtually_split: - expected_block_len = expected_tensor_size // num_blocks // 2 - else: - expected_block_len = expected_tensor_size // num_blocks + expected_block_len = expected_tensor_size // num_blocks for i, block_entry in enumerate(blocks_data): block_start_addr, block_len, tp_rank = block_entry diff --git a/tests/v1/kv_connector/unit/test_transfer_topology_sharded.py b/tests/v1/kv_connector/unit/test_transfer_topology_sharded.py index ac00bb4812..1fed8ff3b7 100644 --- a/tests/v1/kv_connector/unit/test_transfer_topology_sharded.py +++ b/tests/v1/kv_connector/unit/test_transfer_topology_sharded.py @@ -18,8 +18,8 @@ class _FakeAttentionBackend: block_size: int, num_kv_heads: int, head_size: int, - ) -> tuple[int, int, int, int, int]: - return (2, num_blocks, num_kv_heads, block_size, head_size) + ) -> tuple[int, int, int, int]: + return (num_blocks, num_kv_heads, block_size, 2 * head_size) def _make_topology( diff --git a/tests/v1/worker/test_gpu_model_runner.py b/tests/v1/worker/test_gpu_model_runner.py index 6e5b1aa5b8..57931669c5 100644 --- a/tests/v1/worker/test_gpu_model_runner.py +++ b/tests/v1/worker/test_gpu_model_runner.py @@ -795,9 +795,10 @@ def test_kv_cache_stride_order(monkeypatch, model_runner): ) # TODO mla test - default_stride = tuple(range(5)) + default_stride = tuple(range(len(expected_kv_cache_shape))) + non_default_stride = (*default_stride[1:], default_stride[0]) # Permutation that gets you back to expected kv shape - for test_stride in ((1, 4, 0, 2, 3), (0, 1, 2, 3, 4)): + for test_stride in (non_default_stride, default_stride): def rnd_stride_order( include_num_layers_dimension: bool = False, test_stride=test_stride @@ -1300,9 +1301,10 @@ def test_hybrid_attention_mamba_tensor_shapes(): actual_kv = vllm_ctx[layer].kv_cache[kernel_block, :] expected = attn_blocks_constant[i] - # Check K and V separately - assert torch.equal(actual_kv[0], expected) - assert torch.equal(actual_kv[1], expected) + # Packed layout: (num_kv_heads, block_size, 2*head_size). Every + # head in the block was filled with the same constant. + for head_idx in range(actual_kv.shape[0]): + assert torch.equal(actual_kv[head_idx], expected) for layer in [layer_2, layer_3, layer_4, layer_5]: for i, kv_block in enumerate(kv_blocks_for_mamba): diff --git a/vllm/distributed/kv_transfer/kv_connector/utils.py b/vllm/distributed/kv_transfer/kv_connector/utils.py index 71c9db075c..77f043ef52 100644 --- a/vllm/distributed/kv_transfer/kv_connector/utils.py +++ b/vllm/distributed/kv_transfer/kv_connector/utils.py @@ -10,7 +10,12 @@ from typing import TYPE_CHECKING, Any, Literal, cast import torch -from vllm.config import VllmConfig, get_current_vllm_config, get_layers_from_vllm_config +from vllm.config import ( + VllmConfig, + get_current_vllm_config, + get_layers_from_vllm_config, + set_current_vllm_config, +) from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory from vllm.logger import init_logger from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase @@ -344,14 +349,15 @@ def get_current_attn_backends( ) from vllm.v1.attention.selector import get_attn_backend - return [ - get_attn_backend( - head_size=vllm_config.model_config.get_head_size(), - dtype=vllm_config.model_config.dtype, - kv_cache_dtype=vllm_config.cache_config.cache_dtype, - use_mla=vllm_config.model_config.use_mla, - ) - ] + with set_current_vllm_config(vllm_config): + return [ + get_attn_backend( + head_size=vllm_config.model_config.get_head_size(), + dtype=vllm_config.model_config.dtype, + kv_cache_dtype=vllm_config.cache_config.cache_dtype, + use_mla=vllm_config.model_config.use_mla, + ) + ] def get_current_attn_backend( @@ -426,12 +432,16 @@ class TransferTopology: head_size=1, ) logger.debug("Test kv_cache_shape: %s", kv_cache_shape) - # Non-MLA backends caches have 5 dims [num_blocks, 2, H,N,D], - # we just mock num_blocks to 1 for the dimension check below. - # Hybrid SSM models assume a single blocks_first layout - self._is_kv_layout_blocks_first = self.is_mamba or ( - len(kv_cache_shape) == 5 and kv_cache_shape[0] == 1 - ) + assert kv_cache_shape[0] == 1, ( + "KV cache layout must be blocks-first; expected mocked " + f"num_blocks=1 in leading dim, got shape {kv_cache_shape}." + ) + if not self.is_mla: + assert len(kv_cache_shape) == 4, ( + "Attention KV cache layout must be standardized as " + "[num_blocks, num_kv_heads, block_size, content_size], " + f"got shape {kv_cache_shape}." + ) self._cross_layers_blocks = False if self.tensor_shape is not None: @@ -490,29 +500,19 @@ class TransferTopology: # Layout properties # ============================================================ - @property - def is_kv_layout_blocks_first(self) -> bool: - return self._is_kv_layout_blocks_first - @property def cross_layers_blocks(self) -> bool: return self._cross_layers_blocks @property def virtually_split_kv_in_blocks(self) -> bool: - # Whether to logically split each block into K and V halves. - # Applies when K/V are interleaved within each block (blocks-first), - # but NOT when cross-layer blocks are used — cross-layer blocks have - # per-layer K/V interleaving (L0_K, L0_V, L1_K, L1_V, ...) so a - # simple half-split does not separate K from V. - return self._is_kv_layout_blocks_first and not self._cross_layers_blocks - - @property - def split_k_and_v(self) -> bool: - # Whether to register regions for K and V separately (when present). - return not ( - self._cross_layers_blocks or self.is_mla or self.is_kv_layout_blocks_first - ) + # Whether to logically split each block into two separately-indexable + # sub-regions. With K and V packed into the content dim, an attention + # block transfers as a single unit — no K/V sub-split is needed. Only + # Mamba still needs this, to index its two state regions (conv/ssm) + # separately. Not applicable to cross-layer blocks (per-layer + # interleaving means a simple half-split does not separate the parts). + return self.is_mamba and not self._cross_layers_blocks # ============================================================ # Common methods @@ -616,8 +616,9 @@ class TransferTopology: # Swap [2<>num_blocks] dims for hybrid SSM layout. cache = cache.transpose(0, 1) - # Regular case: backends like FA register K/V in separate regions - return cache if self.split_k_and_v else [cache] + # K and V are packed into one tensor (content dim), so each layer + # registers as a single region. + return [cache] def describe(self, remote_engine_id: EngineId, remote_pp_rank: int = 0) -> str: """One-line summary of transfer config for logging.""" diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_layout.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_layout.py index 39dbec3886..d80541ade6 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_layout.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_layout.py @@ -7,6 +7,7 @@ from typing import NamedTuple import torch from vllm.v1.kv_cache_interface import ( + AttentionSpec, KVCacheConfig, KVCacheSpec, MLAAttentionSpec, @@ -56,6 +57,11 @@ def is_mla_cache_layer( return isinstance(spec, (MLAAttentionSpec, SlidingWindowMLASpec)) +def _content_packed_dim(spec: AttentionSpec) -> int: + head_size_v = getattr(spec, "head_size_v", spec.head_size) + return spec.head_size + head_size_v + + def _spec_dim_matches(value: int, expected: int | None) -> bool: return expected is None or value == expected @@ -223,6 +229,30 @@ def get_layer_transfer_geometry( split_kv_regions=False, ) + if ( + not is_mla_cache + and isinstance(spec, AttentionSpec) + and len(shape) == 4 + and shape[1] == spec.num_kv_heads + and shape[2] == spec.block_size + and shape[3] == _content_packed_dim(spec) + ): + num_blocks, num_kv_heads, block_size, packed_dim = shape + slot_size_bytes = num_kv_heads * packed_dim * element_size + block_len = block_size * slot_size_bytes + return LayerTransferGeometry( + num_blocks=num_blocks, + block_size=block_size, + block_len=block_len, + slot_size_bytes=slot_size_bytes, + block_stride=stride[0], + local_kv_stride=None, + remote_kv_stride=None, + transfers_per_block=1, + regions_per_block=1, + split_kv_regions=False, + ) + cache_kind = "MLA" if is_mla_cache else "K/V" raise ValueError( f"Unsupported MoRIIO {cache_kind} cache shape for layer " diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py index eeee594cc3..10554e7ede 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py @@ -216,12 +216,10 @@ class NixlBaseConnectorWorker: if n_regions == 0 or self.num_regions == 0: return [False] * num_fa_descs nblk = num_fa_descs // self.num_regions - virtually_split = self.transfer_topo.virtually_split_kv_in_blocks flags: list[bool] = [] for i in range(n_regions): replicated = self._is_region_replicated(i) - num_streams = 1 if replicated or not virtually_split else 2 - flags.extend([replicated] * (num_streams * nblk)) + flags.extend([replicated] * nblk) assert len(flags) == num_fa_descs, ( f"FA desc flags {len(flags)} != num_fa_descs {num_fa_descs}" ) @@ -712,31 +710,31 @@ class NixlBaseConnectorWorker: NOT directly supported by NIXL (e.g., tpu) """ xfer_buffers: dict[str, torch.Tensor] = {} - inv_order = [0, 1, 3, 2, 4] try: for layer_name, kv_cache in kv_caches.items(): kv_shape = kv_cache.shape kv_dtype = kv_cache.dtype permute_shape = False - if ( - self.kv_cache_layout == "NHD" - and self.vllm_config.kv_transfer_config is not None - and self.vllm_config.kv_transfer_config.enable_permute_local_kv - ): - logger.info_once( - "'enable_permute_local_kv' flag is enabled while " - "device KV Layout is NHD. Init host buffer with" - " HND to better support Decode/Prefill TP_ratio > 1." - ) - # Since NHD will not support Decode/Prefill TP_ratio > 1, - # we can leverage host_buffer for permute - self.host_buffer_kv_cache_layout = "HND" - kv_shape = ( - tuple(kv_shape[i] for i in inv_order) - if not self.use_mla - else kv_shape - ) - permute_shape = not self.use_mla + inv_order = (0, 2, 1, 3) + if not self.use_mla: + assert kv_cache.ndim == 4 + + if self.kv_cache_layout == "NHD": + if self.kv_transfer_config.enable_permute_local_kv: + logger.info_once( + "'enable_permute_local_kv' flag is enabled while " + "device KV Layout is NHD. Init host buffer with" + " HND to better support Decode/Prefill TP_ratio > 1." + ) + # Since NHD will not support Decode/Prefill TP_ratio > 1, + # we can leverage host_buffer for permute. + self.host_buffer_kv_cache_layout = "HND" + else: + # Packed KV layout is logical (B, H, N, 2*D). Allocate + # (B, N, H, 2*D) and view it as logical (B, H, N, 2*D) + # so raw NIXL transfers see NHD physical strides. + kv_shape = tuple(kv_shape[i] for i in inv_order) + permute_shape = True xfer_buffers[layer_name] = torch.empty( kv_shape, dtype=kv_dtype, device="cpu" @@ -1151,9 +1149,7 @@ class NixlBaseConnectorWorker: f"backend={self.backend_name}, " "all_backends=" f"{[backend.get_name() for backend in self.attn_backends]}, " - f"kv_cache_layout={self.kv_cache_layout}, " - "blocks_first=" - f"{self.transfer_topo.is_kv_layout_blocks_first}" + f"kv_cache_layout={self.kv_cache_layout}" ) # Need to make sure the device ID is non-negative for NIXL, @@ -1184,22 +1180,6 @@ class NixlBaseConnectorWorker: regions_per_layer = self.num_regions // num_local_layers self._remote_region_offset = regions_per_layer * start_layer - if self.transfer_topo.virtually_split_kv_in_blocks: - # NOTE (NickLucche) When FlashInfer is used, memory is registered - # with joint KV for each block. This minimizes the overhead in - # registerMem allowing faster descs queries. In order to be able to - # split on kv_heads dim as required by heterogeneous TP, one must - # be able to index K/V separately. Hence we double the number - # of 'virtual' regions here and halve `block_len` below. - # Similarly for Mamba layers, we register SSM+Conv as a single region and - # then duplicate it logically to be able to index SSM/Conv separately. - # Exception: key-only REPLICATE regions (MLA) have no V half, so - # they contribute a single desc stream and are not doubled. - self.num_regions = sum( - 1 if self._is_region_replicated(i) else 2 - for i in range(len(self._region_is_mla)) - ) - # Total local FA descriptors (boundary between FA and mamba descs). self.num_descs = self.num_regions * self.num_blocks @@ -1363,22 +1343,6 @@ class NixlBaseConnectorWorker: block_offset = block_id * page_stride addr = base_addr + block_offset result.append((addr, kv_block_len, self.device_id)) - - if ( - self.transfer_topo.virtually_split_kv_in_blocks - and not self._is_region_replicated(i) - ): - # Separate and interleave K/V regions to maintain the same - # descs ordering. This is needed for selecting contiguous heads - # when split across TP ranks. (Skipped for key-only REPLICATE.) - second_split = self.get_backend_aware_kv_block_len( - layer_idx=i, first_split=False, mamba_view=False - ) - for block_id in range(num_blocks): - block_offset = block_id * page_stride - addr = base_addr + block_offset - v_addr = addr + kv_block_len - result.append((v_addr, second_split, self.device_id)) return result def _build_fa_remote( @@ -1423,20 +1387,6 @@ class NixlBaseConnectorWorker: # tp rank of size local_block_len. addr = base_addr + block_offset + rank_offset result.append((addr, local_block_len, nixl_agent_meta.device_id)) - - emits_v = self.transfer_topo.virtually_split_kv_in_blocks and not replicated - if emits_v: - # With FlashInfer index V separately to allow head splitting. - second_split = self.get_backend_aware_kv_block_len( - layer_idx=i, first_split=False, mamba_view=False - ) - second_split = second_split // num_reads - for block_id in range(num_blocks): - block_offset = block_id * page_size - addr = base_addr + block_offset + rank_offset - # Hop over the first split of remote page, K, to read V. - v_addr = addr + nixl_agent_meta.block_lens[i] // 2 - result.append((v_addr, second_split, nixl_agent_meta.device_id)) return result def register_local_xfer_handler( @@ -1927,24 +1877,18 @@ class NixlBaseConnectorWorker: block_size_ratio, ) - split_k_and_v = self.transfer_topo.split_k_and_v - for block_ids in block_ids_list: indices = torch.tensor(block_ids, device=self.device_type, dtype=torch.long) - for _, cache_or_caches in self.device_kv_caches.items(): - cache_list = cache_or_caches if split_k_and_v else [cache_or_caches] - for cache in cache_list: - if self.enable_permute_local_kv and block_size_ratio > 1: - kv_postprocess_blksize_and_layout_on_receive( - cache, indices, block_size_ratio - ) - elif self.enable_permute_local_kv: - kv_postprocess_layout_on_receive(cache, indices) - else: - kv_postprocess_blksize_on_receive( - cache, indices, block_size_ratio - ) + for cache in self.device_kv_caches.values(): + if self.enable_permute_local_kv and block_size_ratio > 1: + kv_postprocess_blksize_and_layout_on_receive( + cache, indices, block_size_ratio + ) + elif self.enable_permute_local_kv: + kv_postprocess_layout_on_receive(cache, indices) + else: + kv_postprocess_blksize_on_receive(cache, indices, block_size_ratio) def post_process_device_kv_on_receive_heterogeneous_attn( self, block_ids: list[int] @@ -2398,12 +2342,10 @@ class NixlBaseConnectorWorker: |1st_split-2nd_split| |1st_split-2nd_split | """ assert self.transfer_topo is not None - virtually_split = self.transfer_topo.virtually_split_kv_in_blocks - if virtually_split and mamba_view: + if self.transfer_topo.virtually_split_kv_in_blocks and mamba_view: block_len = self._mamba_ssm_size[not first_split] else: - half_block = virtually_split and not self._is_region_replicated(layer_idx) - block_len = self.block_len_per_layer[layer_idx] // (2 if half_block else 1) + block_len = self.block_len_per_layer[layer_idx] return block_len def get_kv_connector_stats(self) -> KVConnectorStats | None: diff --git a/vllm/models/minimax_m3/common/ops/sparse_attn.py b/vllm/models/minimax_m3/common/ops/sparse_attn.py index 08d375dc61..e32818c476 100644 --- a/vllm/models/minimax_m3/common/ops/sparse_attn.py +++ b/vllm/models/minimax_m3/common/ops/sparse_attn.py @@ -8,7 +8,8 @@ equal the sparse block size (128), so one selected block maps to exactly one page. Main K/V cache layout (vLLM): - ``(num_blocks, 2, 128, num_kv_heads, head_dim)`` K=[:,0] V=[:,1] + ``(num_blocks, num_kv_heads, 128, 2 * head_dim)`` + K=[..., :head_dim] V=[..., head_dim:] Only the paths MiniMax M3 uses are implemented: no attention sink, base-2 (exp2/log2) softmax. The decode kernels use split-K (flash-decoding) over the @@ -50,7 +51,7 @@ _FP8_DTYPES = ( @triton.jit(do_not_specialize_on_alignment=["seq_lens", "prefix_lens"]) def _gqa_sparse_fwd_kernel( q_ptr, # [total_q, num_heads, head_dim] - kv_cache_ptr, # main cache: [num_blocks, 2, 128, num_kv_heads, head_dim] + kv_cache_ptr, # main cache: [num_blocks, num_kv_heads, 128, 2*head_dim] t_ptr, # topk_idx: [num_kv_heads, total_q, topk] o_ptr, # [total_q, num_heads, head_dim] block_table_ptr, # [num_reqs, max_blocks] @@ -68,9 +69,8 @@ def _gqa_sparse_fwd_kernel( stride_qh, stride_qd, stride_kv_blk, - stride_kv_kv, - stride_kv_pos, stride_kv_h, + stride_kv_pos, stride_kv_d, stride_th, stride_tn, @@ -140,9 +140,8 @@ def _gqa_sparse_fwd_kernel( k = tl.load( kv_cache_ptr + page * stride_kv_blk - + 0 * stride_kv_kv - + off_n[None, :] * stride_kv_pos + pid_kh * stride_kv_h + + off_n[None, :] * stride_kv_pos + off_d[:, None] * stride_kv_d, mask=d_mask[:, None] & pos_mask[None, :], other=0.0, @@ -162,10 +161,9 @@ def _gqa_sparse_fwd_kernel( v = tl.load( kv_cache_ptr + page * stride_kv_blk - + 1 * stride_kv_kv - + off_n[:, None] * stride_kv_pos + pid_kh * stride_kv_h - + off_d[None, :] * stride_kv_d, + + off_n[:, None] * stride_kv_pos + + (head_dim + off_d[None, :]) * stride_kv_d, mask=pos_mask[:, None] & d_mask[None, :], other=0.0, ) @@ -206,7 +204,7 @@ def _gqa_sparse_fwd_kernel( @triton.jit(do_not_specialize=["decode_query_len"]) def _gqa_sparse_decode_kernel( q_ptr, # [total_q, num_heads, head_dim] - kv_cache_ptr, # main cache: [num_blocks, 2, 128, num_kv_heads, head_dim] + kv_cache_ptr, # main cache: [num_blocks, num_kv_heads, 128, 2*head_dim] t_ptr, # topk_idx: [num_kv_heads, total_q, topk] o_ptr, # partial out: [NUM_TOPK_CHUNKS, total_q, num_heads, head_dim] lse_ptr, # partial lse (log2): [NUM_TOPK_CHUNKS, total_q, num_heads] @@ -222,9 +220,8 @@ def _gqa_sparse_decode_kernel( stride_qh, stride_qd, stride_kv_blk, - stride_kv_kv, - stride_kv_pos, stride_kv_h, + stride_kv_pos, stride_kv_d, stride_th, stride_tn, @@ -300,9 +297,8 @@ def _gqa_sparse_decode_kernel( k = tl.load( kv_cache_ptr + page * stride_kv_blk - + 0 * stride_kv_kv - + off_n[None, :] * stride_kv_pos + pid_kh * stride_kv_h + + off_n[None, :] * stride_kv_pos + off_d[:, None] * stride_kv_d, mask=d_mask[:, None] & pos_mask[None, :], other=0.0, @@ -319,10 +315,9 @@ def _gqa_sparse_decode_kernel( v = tl.load( kv_cache_ptr + page * stride_kv_blk - + 1 * stride_kv_kv - + off_n[:, None] * stride_kv_pos + pid_kh * stride_kv_h - + off_d[None, :] * stride_kv_d, + + off_n[:, None] * stride_kv_pos + + (head_dim + off_d[None, :]) * stride_kv_d, mask=pos_mask[:, None] & d_mask[None, :], other=0.0, ) @@ -418,7 +413,7 @@ def _merge_topk_attn_out_kernel( @torch.no_grad() def minimax_m3_sparse_attn( q: torch.Tensor, # [total_q, num_heads, head_dim] - kv_cache: torch.Tensor, # [num_blocks, 2, 128, num_kv_heads, head_dim] + kv_cache: torch.Tensor, # [num_blocks, num_kv_heads, 128, 2*head_dim] topk_idx: torch.Tensor, # [num_kv_heads, total_q, topk] block_table: torch.Tensor, # [batch, max_blocks] cu_seqlens_q: torch.Tensor, # [batch+1] int32 @@ -459,7 +454,6 @@ def minimax_m3_sparse_attn( kv_cache.stride(1), kv_cache.stride(2), kv_cache.stride(3), - kv_cache.stride(4), topk_idx.stride(0), topk_idx.stride(1), topk_idx.stride(2), @@ -476,7 +470,7 @@ def minimax_m3_sparse_attn( @torch.no_grad() def minimax_m3_sparse_attn_decode( q: torch.Tensor, # [total_q, num_heads, head_dim] - kv_cache: torch.Tensor, # [num_blocks, 2, 128, num_kv_heads, head_dim] + kv_cache: torch.Tensor, # [num_blocks, num_kv_heads, 128, 2*head_dim] topk_idx: torch.Tensor, # [num_kv_heads, total_q, topk] block_table: torch.Tensor, # [num_reqs, max_blocks] seq_lens: torch.Tensor, # [num_reqs] int32 @@ -529,7 +523,6 @@ def minimax_m3_sparse_attn_decode( kv_cache.stride(1), kv_cache.stride(2), kv_cache.stride(3), - kv_cache.stride(4), topk_idx.stride(0), topk_idx.stride(1), topk_idx.stride(2), diff --git a/vllm/models/minimax_m3/common/sparse_attention.py b/vllm/models/minimax_m3/common/sparse_attention.py index 88109772eb..c87fe74c17 100644 --- a/vllm/models/minimax_m3/common/sparse_attention.py +++ b/vllm/models/minimax_m3/common/sparse_attention.py @@ -104,20 +104,25 @@ class MiniMaxM3SparseBackend(AttentionBackend): head_size: int, cache_dtype_str: str = "auto", ) -> tuple[int, ...]: - return (num_blocks, 2, block_size, num_kv_heads, head_size) + # K and V are packed into the content dim: logical (B, H, N, 2*hs). + return (num_blocks, num_kv_heads, block_size, 2 * head_size) @staticmethod def get_kv_cache_stride_order( include_num_layers_dimension: bool = False, ) -> tuple[int, ...]: - # Permutation from get_kv_cache_shape to the actual memory layout. + # `stride_order` indicates the permutation that gets us from + # `get_kv_cache_shape` (logical (B, H, N, 2*hs)) to the actual memory + # layout we want. if include_num_layers_dimension: raise NotImplementedError # no cross-layer KV blocks in M3 cache_layout = get_kv_cache_layout() if cache_layout == "NHD": - stride_order = (0, 1, 2, 3, 4) + # (num_blocks, block_size, num_kv_heads, 2*head_size) + stride_order = (0, 2, 1, 3) elif cache_layout == "HND": - stride_order = (0, 1, 3, 2, 4) + # (num_blocks, num_kv_heads, block_size, 2*head_size) + stride_order = (0, 1, 2, 3) else: raise ValueError(f"Unknown cache layout format {cache_layout}.") return stride_order diff --git a/vllm/models/minimax_m3/nvidia/sparse_attention_msa.py b/vllm/models/minimax_m3/nvidia/sparse_attention_msa.py index 545667c8cf..cf56fcf5e5 100644 --- a/vllm/models/minimax_m3/nvidia/sparse_attention_msa.py +++ b/vllm/models/minimax_m3/nvidia/sparse_attention_msa.py @@ -79,8 +79,7 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl): # strided view directly (topK stays innermost-contiguous). prefill_topk = topk[nd:num_tokens].transpose(0, 1) qp = q[nd:] - k_cache = kv_cache[:, 0].transpose(1, 2) - v_cache = kv_cache[:, 1].transpose(1, 2) + k_cache, v_cache = kv_cache.split(self.head_size, dim=-1) k2q_row_ptr, k2q_q_indices, schedule = build_k2q_csr( prefill_topk, p.cu_seqlens_q, diff --git a/vllm/platforms/rocm.py b/vllm/platforms/rocm.py index 6c3a0fe96e..4b182aeb88 100644 --- a/vllm/platforms/rocm.py +++ b/vllm/platforms/rocm.py @@ -427,8 +427,8 @@ def _get_backend_priorities( ] backends = [] - # ROCM_ATTN uses (2, num_blocks, ...) KV cache layout which is - # incompatible with KV connectors that require blocks-first layout. + # Keep ROCM_ATTN disabled for KV connectors until connector transfer + # semantics are validated for its asymmetric native K/V cache views. if not use_kv_connector: backends.append(AttentionBackendEnum.ROCM_ATTN) if rocm_aiter_ops.is_mha_enabled(): @@ -511,6 +511,20 @@ class RocmPlatform(Platform): attn_selector_config.use_sparse, attn_selector_config.use_kv_connector, ) + from vllm.config import get_current_vllm_config_or_none + + vllm_config = get_current_vllm_config_or_none() + is_encoder_decoder = ( + getattr(getattr(vllm_config, "model_config", None), "attn_type", None) + == "encoder_decoder" + ) + # ROCM_ATTN still uses a legacy attention layout (KV is the outer + # dimension) that is incompatible with the encoder backend layouts. The + # encoder and decoder need the layouts to match. This is currently + # enforced implicitly. + # TODO: Make this explicit in the selector in a future PR. + if is_encoder_decoder and AttentionBackendEnum.ROCM_ATTN in backend_priorities: + backend_priorities.remove(AttentionBackendEnum.ROCM_ATTN) for priority, backend in enumerate(backend_priorities): try: backend_class = backend.get_class() diff --git a/vllm/utils/torch_utils.py b/vllm/utils/torch_utils.py index cf821a54ba..1472024c2b 100644 --- a/vllm/utils/torch_utils.py +++ b/vllm/utils/torch_utils.py @@ -416,26 +416,24 @@ def nvfp4_kv_cache_full_dim(head_size: int) -> int: return head_size // 2 + head_size // 16 -def _nvfp4_split_data_scale( +def nvfp4_split_data_scale( kv_side: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: - """Split a single NVFP4 KV-side buffer into data and scale views. + """Split one side (K or V) of an NVFP4 KV cache into data and scale. - The input is a 4D tensor for one KV side (K or V) whose last - dimension is ``full_dim = data_dim + scale_dim``. The physical - layout within each side is [data | scale], both packed contiguously. + The input is a 4D uint8 tensor whose last dimension is + ``full_dim = data_dim + scale_dim``. The physical layout within each + side is ``[data | scale]``, both packed contiguously. + + The caller is responsible for slicing K and V from the combined cache + first (e.g. ``kv_cache.split(num_kv_heads, dim=1)``). Args: - kv_side: 4D uint8 tensor with shape - ``(num_pages, dim_1, dim_2, full_dim)``. - May be in any permutation order (NHD or HND). + kv_side: 4D uint8 tensor ``(B, H, N, full_dim)``. Returns: - ``(data, scale)`` where - ``data`` is a uint8 view with shape - ``(num_pages, dim_1, dim_2, data_dim)``. - ``scale`` is a float8_e4m3fn view with shape - ``(num_pages, dim_1, dim_2, scale_dim)``. + ``(data, scale)`` where *data* is uint8 and *scale* is + float8_e4m3fn, both views of the same storage. """ num_pages = kv_side.shape[0] dim_1, dim_2 = kv_side.shape[1], kv_side.shape[2] @@ -468,38 +466,6 @@ def _nvfp4_split_data_scale( return data, scale -def nvfp4_kv_cache_split_views(kv_cache: torch.Tensor) -> tuple[tuple, tuple]: - """Split an NVFP4 KV cache tensor into data and scale views. - - Accepts either a 5D tensor ``(num_pages, 2, dim_2, dim_3, full_dim)`` - or a 4D single-side tensor ``(num_pages, dim_2, dim_3, full_dim)``. - - Per-page layout: [K_data | K_scale | V_data | V_scale]. - Each KV side is self-contained (data followed by its scale), so the - 5D case simply splits each side independently. - - The returned views are in the same dim order as the input (NHD or - HND), so callers get views matching whichever order they passed in. - - Args: - kv_cache: 5D or 4D uint8 tensor where the last dimension is - ``full_dim = data_dim + scale_dim = 9 * head_size / 16``. - - Returns: - For 5D input: - ``(k_data, v_data), (k_scale, v_scale)`` - For 4D input (single KV side): - ``(data,), (scale,)`` - """ - if kv_cache.dim() == 4: - data, scale = _nvfp4_split_data_scale(kv_cache) - return (data,), (scale,) - - k_data, k_scale = _nvfp4_split_data_scale(kv_cache[:, 0]) - v_data, v_scale = _nvfp4_split_data_scale(kv_cache[:, 1]) - return (k_data, v_data), (k_scale, v_scale) - - def create_kv_caches_with_random_flash( num_blocks: int, block_size: int, diff --git a/vllm/v1/attention/backend.py b/vllm/v1/attention/backend.py index ec07023f80..4ecf0b9ad0 100644 --- a/vllm/v1/attention/backend.py +++ b/vllm/v1/attention/backend.py @@ -122,11 +122,11 @@ class AttentionBackend(ABC): ) -> tuple[int, ...]: """ Get the physical (memory layout) ordering of the kv cache dimensions. - e.g. if the KV cache shape is - [2, num_blocks, block_size, num_heads, head_size], - and get_kv_cache_stride_order returns (1, 3, 0, 2, 4) then the physical + Standard attention backends pack K and V into the content dim, giving + the logical shape [num_blocks, num_heads, block_size, 2 * head_size]. + e.g. if get_kv_cache_stride_order returns (0, 2, 1, 3) then the physical ordering of dimensions is - [num_blocks, num_heads, 2, block_size, head_size]. + [num_blocks, block_size, num_heads, 2 * head_size]. If this function is unimplemented / raises NotImplementedError, the physical layout of the KV cache will match the logical shape. @@ -135,9 +135,9 @@ class AttentionBackend(ABC): include_num_layers_dimension: if True, includes an additional num_layers dimension, which is assumed to be prepended to the logical KV cache shape. - With the above example, a return value (2, 4, 0, 1, 3, 5) + With the above example, a return value (1, 0, 3, 2, 4) corresponds to - [num_blocks, num_heads, num_layers, 2, block_size, head_size]. + [num_blocks, num_layers, block_size, num_heads, 2 * head_size]. If an additional dimension is NOT included in the returned tuple, the physical layout will not include a layers dimension. diff --git a/vllm/v1/attention/backends/flash_attn.py b/vllm/v1/attention/backends/flash_attn.py index d83d2f4d81..844385326f 100755 --- a/vllm/v1/attention/backends/flash_attn.py +++ b/vllm/v1/attention/backends/flash_attn.py @@ -133,25 +133,29 @@ class FlashAttentionBackend(AttentionBackend): ) -> tuple[int, ...]: if block_size % 16 != 0: raise ValueError("Block size must be a multiple of 16.") - return (num_blocks, 2, block_size, num_kv_heads, head_size) + # K and V are packed into the content dim: logical (B, H, N, 2*D). + return (num_blocks, num_kv_heads, block_size, 2 * head_size) @staticmethod def get_kv_cache_stride_order( include_num_layers_dimension: bool = False, ) -> tuple[int, ...]: - # `stride_order` indicates the permutation that gets - # us from `get_kv_cache_shape` to the actual memory layout we want. + # `stride_order` indicates the permutation that gets us from + # `get_kv_cache_shape` (logical (B, H, N, 2*D)) to the actual memory + # layout we want. cache_layout = get_kv_cache_layout() if cache_layout == "NHD" and include_num_layers_dimension: - # (num_blocks, num_layers, 2, block_size, num_kv_heads, head_size) - return (1, 0, 2, 3, 4, 5) + # (num_blocks, num_layers, block_size, num_kv_heads, 2*head_size) + return (1, 0, 3, 2, 4) elif cache_layout == "NHD": - stride_order = (0, 1, 2, 3, 4) + # (num_blocks, block_size, num_kv_heads, 2*head_size) + stride_order = (0, 2, 1, 3) elif cache_layout == "HND" and include_num_layers_dimension: - # (num_blocks, num_kv_heads, num_layers, 2, block_size, head_size) - return (1, 4, 0, 2, 3, 5) + # (num_blocks, num_kv_heads, num_layers, block_size, 2*head_size) + return (1, 2, 0, 3, 4) elif cache_layout == "HND": - stride_order = (0, 1, 3, 2, 4) + # (num_blocks, num_kv_heads, block_size, 2*head_size) + stride_order = (0, 1, 2, 3) else: raise ValueError(f"Unknown cache layout format {cache_layout}.") return stride_order @@ -819,7 +823,7 @@ class FlashAttentionImpl(AttentionImpl): key: shape = [num_tokens, num_kv_heads, head_size] value: shape = [num_tokens, num_kv_heads, head_size] kv_cache: shape = - [num_blocks, 2, block_size, num_kv_heads, head_size] + [num_blocks, num_kv_heads, block_size, 2 * head_size] attn_metadata: Metadata for attention. Returns: shape = [num_tokens, num_heads * head_size] @@ -866,8 +870,8 @@ class FlashAttentionImpl(AttentionImpl): layer, ) - # For decoder and cross-attention, use KV cache as before - key_cache, value_cache = kv_cache.unbind(1) + # (B, H, N, 2*D) -> ((B, N, H, D), (B, N, H, D)) + key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1) # Fix degenerate strides on size-1 dims (e.g. num_kv_heads=1 with TP). # FA3/4 on H100+ uses TMA, which requires ≥16-byte stride alignment. # See vllm.utils.torch_utils.canonicalize_singleton_dim_strides. @@ -1075,7 +1079,8 @@ class FlashAttentionImpl(AttentionImpl): # Scatter write into the KV cache using slot_mapping indices. # No TMA kernel is invoked here, so stride canonicalization is not needed. - key_cache, value_cache = kv_cache.unbind(1) + # (B, H, N, 2*D) -> ((B, N, H, D), (B, N, H, D)) + key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1) # Reshape the input keys and values and store them in the cache. # Skip this if sharing KV cache with an earlier attention layer. diff --git a/vllm/v1/attention/backends/flash_attn_diffkv.py b/vllm/v1/attention/backends/flash_attn_diffkv.py index ff8fbfc022..02c88b4b36 100644 --- a/vllm/v1/attention/backends/flash_attn_diffkv.py +++ b/vllm/v1/attention/backends/flash_attn_diffkv.py @@ -83,10 +83,12 @@ class FlashAttentionDiffKVBackend(FlashAttentionBackend): ) -> tuple[int, ...]: if block_size % 16 != 0: raise ValueError("Block size must be a multiple of 16.") + # Logical (blocks-first, head-major) layout: K and V (with their + # different head sizes) packed in the content dim. return ( num_blocks, - block_size, num_kv_heads, + block_size, head_size + FlashAttentionDiffKVBackend.head_size_v, ) @@ -94,21 +96,22 @@ class FlashAttentionDiffKVBackend(FlashAttentionBackend): def get_kv_cache_stride_order( include_num_layers_dimension: bool = False, ) -> tuple[int, ...]: - # `stride_order` indicates the permutation that gets - # us from `get_kv_cache_shape` to the actual memory layout we want. + # `stride_order` indicates the permutation that gets us from + # `get_kv_cache_shape` (logical (B, H, N, C_k+C_v)) to the actual + # memory layout we want. cache_layout = get_kv_cache_layout() if cache_layout == "NHD" and include_num_layers_dimension: - # (num_blocks, num_layers, block_size, - # num_kv_heads, head_size + head_size_v) - return (1, 0, 2, 3, 4) + # (num_blocks, num_layers, block_size, num_kv_heads, C_k+C_v) + return (1, 0, 3, 2, 4) elif cache_layout == "NHD": - stride_order = (0, 1, 2, 3) - elif cache_layout == "HND" and include_num_layers_dimension: - # (num_blocks, num_kv_heads, num_layers, - # block_size, head_size + head_size_v) - return (1, 3, 0, 2, 4) - elif cache_layout == "HND": + # (num_blocks, block_size, num_kv_heads, C_k+C_v) stride_order = (0, 2, 1, 3) + elif cache_layout == "HND" and include_num_layers_dimension: + # (num_blocks, num_kv_heads, num_layers, block_size, C_k+C_v) + return (1, 2, 0, 3, 4) + elif cache_layout == "HND": + # (num_blocks, num_kv_heads, block_size, C_k+C_v) + stride_order = (0, 1, 2, 3) else: raise ValueError(f"Unknown cache layout format {cache_layout}.") return stride_order @@ -142,21 +145,14 @@ class FlashAttentionDiffKVImpl(FlashAttentionImpl): # we use direct Q, K, V tensors without caching return - # Unlike standard FlashAttn which splits kv_cache via unbind(0), # DiffKV packs K and V into a single tensor along the last dim: # kv_cache shape: [num_blocks, block_size, num_kv_heads, # head_size_k + head_size_v] - # The triton kernel handles this combined layout directly. - # - # NOTE(woosuk): key and value are padded while slot_mapping is - # not padded. However, we don't need to do key[:num_actual_tokens] - # and value[:num_actual_tokens] because the reshape_and_cache_flash - # op uses the slot_mapping's shape to determine the number of - # actual tokens. + # (B, H, N, C) -> (B, N, H, C) for kernel compatibility. triton_reshape_and_cache_flash_diffkv( key, value, - kv_cache, + kv_cache.transpose(1, 2), slot_mapping, self.kv_cache_dtype, layer._k_scale, @@ -229,10 +225,8 @@ class FlashAttentionDiffKVImpl(FlashAttentionImpl): layer, ) - # For decoder and cross-attention, use KV cache as before - # Different head_size for K and V - key_cache = kv_cache[..., : self.head_size] - value_cache = kv_cache[..., self.head_size :] + # (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs)) + key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1) # Fix degenerate strides on size-1 dims (e.g. num_kv_heads=1 with TP). # FA3/4 on H100+ uses TMA, which requires ≥16-byte stride alignment. # See vllm.utils.torch_utils.canonicalize_singleton_dim_strides. diff --git a/vllm/v1/attention/backends/flashinfer.py b/vllm/v1/attention/backends/flashinfer.py index 12eab21e3e..699073939e 100755 --- a/vllm/v1/attention/backends/flashinfer.py +++ b/vllm/v1/attention/backends/flashinfer.py @@ -51,7 +51,7 @@ from vllm.utils.torch_utils import ( is_quantized_kv_cache, is_strictly_contiguous, nvfp4_kv_cache_full_dim, - nvfp4_kv_cache_split_views, + nvfp4_split_data_scale, ) from vllm.v1.attention.backend import ( AttentionBackend, @@ -111,9 +111,12 @@ def _trtllm_prefill_attn_kvfp8_dequant( src_stride_page, src_stride_kv, src_stride_head, + src_stride_block, + src_stride_head_size, DST_K_CACHE_STRIDE: tl.constexpr, DST_KV_CACHE_STRIDE: tl.constexpr, HEAD_STRIDE: tl.constexpr, + HEAD_SIZE: tl.constexpr, NUM_KV_HEADS: tl.constexpr, ): batch_idx = tl.program_id(0).to(tl.int64) @@ -129,18 +132,25 @@ def _trtllm_prefill_attn_kvfp8_dequant( v_scale_val = tl.load(v_scale_ptr) mock_page_idx = batch_idx * block_table_stride + mock_block_table_idx + 1 - head_offsets = tl.arange(0, HEAD_STRIDE) + logical_offsets = tl.arange(0, HEAD_STRIDE) + block_offsets = logical_offsets // HEAD_SIZE + head_size_offsets = logical_offsets % HEAD_SIZE for h in range(NUM_KV_HEADS): h_off = tl.cast(h, tl.int64) # Read K from source (supports non-contiguous page/kv/head strides) - src_k = orig_page_num * src_stride_page + h_off * src_stride_head + head_offsets + src_k = ( + orig_page_num * src_stride_page + + h_off * src_stride_head + + block_offsets * src_stride_block + + head_size_offsets * src_stride_head_size + ) fp8_k = tl.load(kv_cache_ptr + src_k) dequant_k = (fp8_k.to(tl.float32) * k_scale_val).to(dequant_dtype) # Write K to contiguous mock cache - dst_k = mock_page_idx * DST_KV_CACHE_STRIDE + h * HEAD_STRIDE + head_offsets + dst_k = mock_page_idx * DST_KV_CACHE_STRIDE + h * HEAD_STRIDE + logical_offsets tl.store(mock_kv_cache_ptr + dst_k, dequant_k) # Read V from source (offset by src_stride_kv for the V half) @@ -148,7 +158,8 @@ def _trtllm_prefill_attn_kvfp8_dequant( orig_page_num * src_stride_page + src_stride_kv + h_off * src_stride_head - + head_offsets + + block_offsets * src_stride_block + + head_size_offsets * src_stride_head_size ) fp8_v = tl.load(kv_cache_ptr + src_v) dequant_v = (fp8_v.to(tl.float32) * v_scale_val).to(dequant_dtype) @@ -158,7 +169,7 @@ def _trtllm_prefill_attn_kvfp8_dequant( mock_page_idx * DST_KV_CACHE_STRIDE + DST_K_CACHE_STRIDE + h * HEAD_STRIDE - + head_offsets + + logical_offsets ) tl.store(mock_kv_cache_ptr + dst_v, dequant_v) @@ -175,17 +186,14 @@ def trtllm_prefill_attn_kvfp8_dequant( assert s[1] == 2 assert dequant_dtype in (torch.bfloat16, torch.float16) + # Logical source layout is (B, 2, H, N, D). The tensor may be a + # non-contiguous view, so the Triton kernel indexes it with actual strides. + strides = kv_cache.stride() num_kv_heads, block_size, head_size = s[2], s[3], s[4] head_stride = block_size * head_size k_cache_stride = num_kv_heads * head_stride kv_cache_stride = k_cache_stride * s[1] - strides = kv_cache.stride() - assert strides[3] == head_size and strides[4] == 1, ( - "For kv cache layouts, (block_size, head_size) " - f"dimensions must be contiguous, got strides {strides}" - ) - new_s = (batch_size * num_of_page_per_token + 1, s[1], s[2], s[3], s[4]) # mock kv cache contains just the pages needed by this prefill mock_kv_cache = torch.empty(new_s, dtype=dequant_dtype, device=kv_cache.device) @@ -207,9 +215,12 @@ def trtllm_prefill_attn_kvfp8_dequant( strides[0], strides[1], strides[2], + strides[3], + strides[4], k_cache_stride, kv_cache_stride, head_stride, + head_size, num_kv_heads, ) return mock_kv_cache, mock_block_table @@ -286,7 +297,7 @@ class BatchDCPPrefillWrapper: self, layer: torch.nn.Module, prefill_query: torch.Tensor, - kv_cache_permute: torch.Tensor, + kv_cache_tuple: tuple[torch.Tensor, torch.Tensor], key: torch.Tensor, value: torch.Tensor, out: torch.Tensor, @@ -296,7 +307,7 @@ class BatchDCPPrefillWrapper: ) output_context_tmp, lse_context_tmp = self._context.run( prefill_query_across_dcp, - kv_cache_permute, + kv_cache_tuple, k_scale=layer._k_scale_float, v_scale=layer._v_scale_float, return_lse=True, @@ -386,28 +397,31 @@ class FlashInferBackend(AttentionBackend): cache_dtype_str: str = "auto", ) -> tuple[int, ...]: if cache_dtype_str == "nvfp4": - # Packed layout: fp4 data + fp8 block scales in last dim - last_dim = nvfp4_kv_cache_full_dim(head_size) - return (num_blocks, 2, block_size, num_kv_heads, last_dim) - return (num_blocks, 2, block_size, num_kv_heads, head_size) + full_dim = nvfp4_kv_cache_full_dim(head_size) + return (num_blocks, 2 * num_kv_heads, block_size, full_dim) + # Pack K and V in the content dim (B, H, N, 2*hs). + return (num_blocks, num_kv_heads, block_size, 2 * head_size) @staticmethod def get_kv_cache_stride_order( include_num_layers_dimension: bool = False, ) -> tuple[int, ...]: # `stride_order` indicates the permutation that gets us from - # `get_kv_cache_shape` to the actual memory layout we want. + # `get_kv_cache_shape` (logical (B, H, N, 2*hs)) to the actual memory + # layout we want. cache_layout = get_kv_cache_layout() if cache_layout == "NHD" and include_num_layers_dimension: - # (num_blocks, num_layers, 2, block_size, num_kv_heads, head_size) - return (1, 0, 2, 3, 4, 5) + # (num_blocks, num_layers, block_size, num_kv_heads, 2*head_size) + return (1, 0, 3, 2, 4) elif cache_layout == "NHD": - stride_order = (0, 1, 2, 3, 4) + # (num_blocks, block_size, num_kv_heads, 2*head_size) + stride_order = (0, 2, 1, 3) elif cache_layout == "HND" and include_num_layers_dimension: - # (num_blocks, 2, num_kv_heads, num_layers, block_size, head_size) - return (1, 2, 4, 0, 3, 5) + # (num_blocks, num_kv_heads, num_layers, block_size, 2*head_size) + return (1, 2, 0, 3, 4) elif cache_layout == "HND": - stride_order = (0, 1, 3, 2, 4) + # (num_blocks, num_kv_heads, block_size, 2*head_size) + stride_order = (0, 1, 2, 3) else: raise ValueError(f"Unknown cache layout format {cache_layout}.") return stride_order @@ -1714,7 +1728,19 @@ class FlashInferImpl(AttentionImpl): if attn_metadata.use_cascade: # Cascade attention (rare case). assert attn_metadata.cascade_wrapper is not None - output.copy_(attn_metadata.cascade_wrapper.run(query, kv_cache)) + stride_order = FlashInferBackend.get_kv_cache_stride_order() + if self.is_kvcache_nvfp4: + kv_cache_views = tuple( + cache.permute(*stride_order) + for cache in kv_cache.split(self.num_kv_heads, dim=1) + ) + else: + kv_perm = kv_cache.permute(*stride_order) + kv_cache_views = kv_perm.split(self.head_size, dim=-1) + kv_tuple = tuple( + canonicalize_singleton_dim_strides(cache) for cache in kv_cache_views + ) + output.copy_(attn_metadata.cascade_wrapper.run(query, kv_tuple)) return output # When using spec decoding, num_decodes can be < num_decode_tokens @@ -1740,14 +1766,23 @@ class FlashInferImpl(AttentionImpl): ) kv_cache_permute = fixed - # For NVFP4, the kv_cache last dim is full_dim (data + scale packed). - # Split into correctly-strided data and scale views. + # Split K/V — zero-copy views. NVFP4 stores K/V as separate head + # groups; other dtypes pack K/V in the content dim. + hs = self.head_size nvfp4_kv_data = None nvfp4_kv_block_scales = None if self.is_kvcache_nvfp4: - nvfp4_kv_data, nvfp4_kv_block_scales = nvfp4_kv_cache_split_views( - kv_cache_permute + k_cache, v_cache = kv_cache.split(self.num_kv_heads, dim=1) + kv_cache_tuple = ( + canonicalize_singleton_dim_strides(k_cache.permute(*stride_order)), + canonicalize_singleton_dim_strides(v_cache.permute(*stride_order)), ) + k_data, k_sf = nvfp4_split_data_scale(kv_cache_tuple[0]) + v_data, v_sf = nvfp4_split_data_scale(kv_cache_tuple[1]) + nvfp4_kv_data = (k_data, v_data) + nvfp4_kv_block_scales = (k_sf, v_sf) + else: + kv_cache_tuple = kv_cache_permute.split(hs, dim=-1) use_dcp = self.dcp_world_size > 1 @@ -1786,7 +1821,7 @@ class FlashInferImpl(AttentionImpl): prefill_wrapper.run( layer, prefill_query, - kv_cache_permute, + kv_cache_tuple, key[num_decode_tokens:], value[num_decode_tokens:], out=output[num_decode_tokens:], @@ -1803,7 +1838,9 @@ class FlashInferImpl(AttentionImpl): assert prefill_wrapper._causal == attn_metadata.causal if self.is_kvcache_nvfp4: - kv_cache_permute = nvfp4_kv_data + kv_cache_for_fi = nvfp4_kv_data + else: + kv_cache_for_fi = kv_cache_tuple kv_cache_sf = ( nvfp4_kv_block_scales if self.is_kvcache_nvfp4 else None ) @@ -1821,7 +1858,7 @@ class FlashInferImpl(AttentionImpl): prefill_wrapper.run( prefill_query, - kv_cache_permute, + kv_cache_for_fi, q_scale=layer._q_scale_float, k_scale=layer._k_scale_float, v_scale=layer._v_scale_float, @@ -1888,8 +1925,7 @@ class FlashInferImpl(AttentionImpl): # TRTLLM prefill attention does not support BF16 Q # and fp8 kv cache. So to enable prefill attention # with fp8 kv cache, we can construct a mock block - # and mock kv cache with BF16 KV involved in the prefill - # + # and mock kv cache with BF16 KV involved in the prefill. kv_cache_permute = canonicalize_singleton_dim_strides( kv_cache_permute ) @@ -1901,15 +1937,21 @@ class FlashInferImpl(AttentionImpl): "KV cache inner dims (block_size, head_size) must be " f"contiguous, got strides {kv_strides}" ) + # fp8 uses (B, H, N, 2*hs); reshape to (B, 2, H, N, hs) + # for the dequant kernel — zero-copy view. The dequant + # kernel handles the interleaved K/V block stride. + B_kv, H_kv, N_kv = kv_cache_permute.shape[:3] + kv_cache_5d = kv_cache_permute.view(B_kv, H_kv, N_kv, 2, hs) + kv_cache_5d = kv_cache_5d.permute(0, 3, 1, 2, 4) mock_kv_cache, mock_block_table = trtllm_prefill_attn_kvfp8_dequant( - kv_cache_permute, + kv_cache_5d, block_tables_prefill, layer._k_scale, layer._v_scale, attn_metadata.q_data_type_prefill, ) else: - mock_kv_cache = kv_cache_permute + mock_kv_cache = kv_cache_tuple mock_block_table = block_tables_prefill trtllm_batch_context_with_kv_cache( @@ -1957,7 +1999,9 @@ class FlashInferImpl(AttentionImpl): assert decode_wrapper._sm_scale == self.scale if self.is_kvcache_nvfp4: - kv_cache_permute = nvfp4_kv_data + kv_cache_for_fi = nvfp4_kv_data + else: + kv_cache_for_fi = kv_cache_tuple kv_cache_sf = nvfp4_kv_block_scales if self.is_kvcache_nvfp4 else None # NVFP4 kernel only supports FP8 output. @@ -1980,7 +2024,7 @@ class FlashInferImpl(AttentionImpl): ) decode_wrapper.run( decode_query, - kv_cache_permute, + kv_cache_for_fi, q_scale=layer._q_scale_float, k_scale=layer._k_scale_float, v_scale=layer._v_scale_float, @@ -1997,7 +2041,7 @@ class FlashInferImpl(AttentionImpl): else: decode_wrapper.run( decode_query, - kv_cache_permute, + kv_cache_for_fi, q_scale=layer._q_scale_float, k_scale=layer._k_scale_float, v_scale=layer._v_scale_float, @@ -2079,7 +2123,7 @@ class FlashInferImpl(AttentionImpl): trtllm_batch_decode_with_kv_cache( query=decode_query, kv_cache=( - nvfp4_kv_data if self.is_kvcache_nvfp4 else kv_cache_permute + nvfp4_kv_data if self.is_kvcache_nvfp4 else kv_cache_tuple ), workspace_buffer=workspace_buffer, block_tables=block_tables_decode, @@ -2119,8 +2163,18 @@ class FlashInferImpl(AttentionImpl): # and value[:num_actual_tokens] because the reshape_and_cache_flash # op uses the slot_mapping's shape to determine the number of # actual tokens. - k_cache = kv_cache[:, 0] - v_cache = kv_cache[:, 1] + if self.is_kvcache_nvfp4: + # (B, 2*H, N, full_dim) -> ((B, N, H, full_dim), + # (B, N, H, full_dim)); + # K heads first, then V heads. + k_cache, v_cache = kv_cache.transpose(1, 2).split( + self.num_kv_heads, dim=-2 + ) + else: + # (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs)) + k_cache, v_cache = kv_cache.transpose(1, 2).split( + self.head_size, dim=-1 + ) torch.ops._C_cache_ops.reshape_and_cache_flash( key, value, diff --git a/vllm/v1/attention/backends/flex_attention.py b/vllm/v1/attention/backends/flex_attention.py index c45294bfc7..b918f2adef 100644 --- a/vllm/v1/attention/backends/flex_attention.py +++ b/vllm/v1/attention/backends/flex_attention.py @@ -130,15 +130,16 @@ class FlexAttentionBackend(AttentionBackend): head_size: int, cache_dtype_str: str = "auto", ) -> tuple[int, ...]: - return (num_blocks, 2, block_size, num_kv_heads, head_size) + # K and V are packed into the content dim: logical (B, H, N, 2*hs). + return (num_blocks, num_kv_heads, block_size, 2 * head_size) @staticmethod def get_kv_cache_stride_order( include_num_layers_dimension: bool = False, ) -> tuple[int, ...]: if include_num_layers_dimension: - return (1, 0, 3, 2, 4, 5) - return (0, 2, 1, 3, 4) + return (1, 0, 3, 2, 4) + return (0, 2, 1, 3) @staticmethod def get_builder_cls() -> type["FlexAttentionMetadataBuilder"]: @@ -1242,7 +1243,8 @@ class FlexAttentionImpl(AttentionImpl): if self.attn_type == AttentionType.ENCODER_ONLY: return - key_cache, value_cache = kv_cache.unbind(1) + # (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs)) + key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1) torch.ops._C_cache_ops.reshape_and_cache_flash( key, value, @@ -1273,7 +1275,7 @@ class FlexAttentionImpl(AttentionImpl): key: shape = [num_tokens, num_kv_heads, head_size] value: shape = [num_tokens, num_kv_heads, head_size] kv_cache: shape = - [num_blocks, 2, block_size, num_kv_heads, head_size] + [num_blocks, num_kv_heads, block_size, 2 * head_size] attn_metadata: Metadata for attention. Returns: shape = [num_tokens, num_heads * head_size] @@ -1347,9 +1349,11 @@ class FlexAttentionImpl(AttentionImpl): else: assert self.attn_type == AttentionType.DECODER - key_cache, value_cache = kv_cache.unbind(1) + kv_cache = kv_cache.transpose(1, 2) + hs = self.head_size + key_cache, value_cache = kv_cache.split(hs, dim=-1) - # Flatten (num_blocks, block_size) into a single token dim + # Flatten (num_blocks, block_size) into a single token dim. key_cache = key_cache.view(-1, self.num_kv_heads, self.head_size) value_cache = value_cache.view(-1, self.num_kv_heads, self.head_size) query, key_tensor, value_tensor = map( diff --git a/vllm/v1/attention/backends/rocm_aiter_fa.py b/vllm/v1/attention/backends/rocm_aiter_fa.py index 0f289649b1..4290c22ced 100644 --- a/vllm/v1/attention/backends/rocm_aiter_fa.py +++ b/vllm/v1/attention/backends/rocm_aiter_fa.py @@ -768,7 +768,8 @@ class AiterFlashAttentionBackend(AttentionBackend): ) -> tuple[int, ...]: if block_size % 16 != 0: raise ValueError("Block size must be a multiple of 16.") - return (num_blocks, 2, block_size, num_kv_heads, head_size) + # K and V are packed into the content dim: logical (B, H, N, 2*hs). + return (num_blocks, num_kv_heads, block_size, 2 * head_size) @classmethod def supports_compute_capability(cls, capability: DeviceCapability) -> bool: @@ -1061,7 +1062,8 @@ class AiterFlashAttentionImpl(AttentionImpl): # Whenever making a change in this method, please benchmark the # performance to make sure it does not introduce any overhead. num_actual_tokens = attn_metadata.num_actual_tokens - key_cache, value_cache = kv_cache.unbind(1) + # (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs)) + key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1) if is_quantized_kv_cache(self.kv_cache_dtype): key_cache = key_cache.view(current_platform.fp8_dtype()) @@ -1388,7 +1390,8 @@ class AiterFlashAttentionImpl(AttentionImpl): kv_cache: torch.Tensor, slot_mapping: torch.Tensor, ): - key_cache, value_cache = kv_cache.unbind(1) + # (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs)) + key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1) # key and value may be None in the case of cross attention. They are # calculated once based on the output from the encoder and then cached @@ -1454,7 +1457,8 @@ class AiterFlashAttentionImpl(AttentionImpl): kv_cache: torch.Tensor, layer_slot_mapping: torch.Tensor, ): - key_cache, value_cache = kv_cache.unbind(1) + # (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs)) + key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1) flash_layout = True is_fp8_kv_cache = is_quantized_kv_cache(self.kv_cache_dtype) diff --git a/vllm/v1/attention/backends/rocm_aiter_unified_attn.py b/vllm/v1/attention/backends/rocm_aiter_unified_attn.py index 57b64cc93d..cc45fade5a 100644 --- a/vllm/v1/attention/backends/rocm_aiter_unified_attn.py +++ b/vllm/v1/attention/backends/rocm_aiter_unified_attn.py @@ -87,7 +87,8 @@ class RocmAiterUnifiedAttentionBackend(RocmAttentionBackend): ) -> tuple[int, ...]: if block_size % 16 != 0: raise ValueError("Block size must be a multiple of 16.") - return (num_blocks, 2, block_size, num_kv_heads, head_size) + # K and V are packed into the content dim: logical (B, H, N, 2*hs). + return (num_blocks, num_kv_heads, block_size, 2 * head_size) @staticmethod def use_cascade_attention(*args, **kwargs) -> bool: @@ -150,10 +151,8 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl): def _split_kv_cache( self, kv_cache: torch.Tensor ) -> tuple[torch.Tensor, torch.Tensor]: - # Blocks-first ``(num_blocks, 2, ...)``. The model runner normalizes any - # shared decoder/cross-attention allocation to this layout, so no - # per-backend restriding is needed here. - return kv_cache.unbind(1) + # (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs)) + return kv_cache.transpose(1, 2).split(self.head_size, dim=-1) def forward( self, diff --git a/vllm/v1/attention/backends/triton_attn.py b/vllm/v1/attention/backends/triton_attn.py index 7b4a652939..fa2fe8f1e6 100644 --- a/vllm/v1/attention/backends/triton_attn.py +++ b/vllm/v1/attention/backends/triton_attn.py @@ -324,6 +324,7 @@ class TritonAttentionBackend(AttentionBackend): ) -> tuple[int, ...]: if block_size % 16 != 0: raise ValueError("Block size must be a multiple of 16.") + # K and V are packed into the content dim: logical (B, H, N, 2*hs). if kv_cache_uses_per_token_head_scales(cache_dtype_str): # Pad the head dim by sizeof(float32)/sizeof(cache_dtype) so the # per-(token, head) scale fits inline after the quantized data; @@ -341,26 +342,30 @@ class TritonAttentionBackend(AttentionBackend): data_head_size = head_size // 2 else: data_head_size = head_size - return (num_blocks, 2, block_size, num_kv_heads, data_head_size + scale_pad) - return (num_blocks, 2, block_size, num_kv_heads, head_size) + padded_hs = data_head_size + scale_pad + return (num_blocks, num_kv_heads, block_size, 2 * padded_hs) + return (num_blocks, num_kv_heads, block_size, 2 * head_size) @staticmethod def get_kv_cache_stride_order( include_num_layers_dimension: bool = False, ) -> tuple[int, ...]: - # `stride_order` indicates the permutation that gets - # us from `get_kv_cache_shape` to the actual memory layout we want. + # `stride_order` indicates the permutation that gets us from + # `get_kv_cache_shape` (logical (B, H, N, 2*hs)) to the actual memory + # layout we want. cache_layout = get_kv_cache_layout() if cache_layout == "NHD" and include_num_layers_dimension: - # (num_blocks, num_layers, 2, block_size, num_kv_heads, head_size) - return (1, 0, 2, 3, 4, 5) + # (num_blocks, num_layers, block_size, num_kv_heads, 2*head_size) + return (1, 0, 3, 2, 4) elif cache_layout == "NHD": - stride_order = (0, 1, 2, 3, 4) + # (num_blocks, block_size, num_kv_heads, 2*head_size) + stride_order = (0, 2, 1, 3) elif cache_layout == "HND" and include_num_layers_dimension: - # (num_blocks, num_kv_heads, num_layers, 2, block_size, head_size) - return (1, 4, 0, 2, 3, 5) + # (num_blocks, num_kv_heads, num_layers, block_size, 2*head_size) + return (1, 2, 0, 3, 4) elif cache_layout == "HND": - stride_order = (0, 1, 3, 2, 4) + # (num_blocks, num_kv_heads, block_size, 2*head_size) + stride_order = (0, 1, 2, 3) else: raise ValueError(f"Unknown cache layout: {cache_layout}") return stride_order @@ -410,12 +415,16 @@ class TritonAttentionImpl(AttentionImpl): _v_scale_cache: torch.Tensor | None = None def _ensure_scale_caches(self, kv_cache: torch.Tensor) -> None: - """Extract per-head scale views from the padded head dimension. + """Extract per-head scale views from the padded content dimension. - The KV cache shape is ``(num_blocks, 2, block_size, nkv, hs+pad)`` - where ``pad = sizeof(float32) / sizeof(cache_dtype)``. The last - ``pad`` elements of each head hold one float32 scale. We create - strided float32 views over those bytes. + The KV cache is packed as logical shape + ``(num_blocks, nkv, block_size, 2 * (hs + pad))`` where + ``pad = sizeof(float32) / sizeof(cache_dtype)``. The content dim holds + ``[K(hs) | K_scale(pad) | V(hs) | V_scale(pad)]`` per (head, slot); the + last ``pad`` elements of each half hold one float32 scale. We create + strided float32 views over those bytes. ``kv_cache`` must be the + packed logical tensor (call before any transpose), but may have HND or + NHD physical strides. Scale shape: ``(num_blocks, block_size, num_kv_heads)`` """ @@ -423,9 +432,10 @@ class TritonAttentionImpl(AttentionImpl): return from vllm.utils.torch_utils import get_dtype_size - num_blocks, _, block_size, nkv, padded_hs = kv_cache.shape + num_blocks, nkv, block_size, content = kv_cache.shape dtype_sz = kv_cache.element_size() scale_pad = get_dtype_size(torch.float32) // dtype_sz # e.g. 4 + padded_hs = content // 2 hs = padded_hs - scale_pad raw = kv_cache.untyped_storage() @@ -433,31 +443,37 @@ class TritonAttentionImpl(AttentionImpl): raw ) - # In the raw bytes, each (block, kv_half, slot, head) occupies - # padded_hs * dtype_sz bytes. The scale float32 sits at byte - # offset hs * dtype_sz within that region. - kv_half_bytes = block_size * nkv * padded_hs * dtype_sz - full_block_f32 = 2 * kv_half_bytes // 4 # stride between blocks - slot_f32 = nkv * padded_hs * dtype_sz // 4 # stride between slots - head_f32 = padded_hs * dtype_sz // 4 # stride between heads - scale_off_f32 = hs * dtype_sz // 4 # offset to scale within head + def to_f32_units(elements: int) -> int: + nbytes = elements * dtype_sz + assert nbytes % 4 == 0 + return nbytes // 4 - # K scales: kv_half=0 + # Actual strides (in float32 units) from the tensor. The logical cache + # may be physically NHD, so do not assume C-contiguous HND layout. + strides = kv_cache.stride() + block_f32 = to_f32_units(strides[0]) + head_f32 = to_f32_units(strides[1]) + slot_f32 = to_f32_units(strides[2]) + # Scale sits at byte offset hs within each (K, then V) content half. + base_off_f32 = to_f32_units(kv_cache.storage_offset()) + k_scale_off_f32 = base_off_f32 + to_f32_units(hs) + v_scale_off_f32 = base_off_f32 + to_f32_units(padded_hs + hs) + + # K scales (first content half) self._k_scale_cache = torch.as_strided( base_f32, size=(num_blocks, block_size, nkv), - stride=(full_block_f32, slot_f32, head_f32), - storage_offset=scale_off_f32, + stride=(block_f32, slot_f32, head_f32), + storage_offset=k_scale_off_f32, ) self._k_scale_cache.fill_(1.0) - # V scales: kv_half=1, offset by kv_half_bytes - v_base_f32 = kv_half_bytes // 4 + # V scales (second content half) self._v_scale_cache = torch.as_strided( base_f32, size=(num_blocks, block_size, nkv), - stride=(full_block_f32, slot_f32, head_f32), - storage_offset=v_base_f32 + scale_off_f32, + stride=(block_f32, slot_f32, head_f32), + storage_offset=v_scale_off_f32, ) self._v_scale_cache.fill_(1.0) @@ -576,7 +592,7 @@ class TritonAttentionImpl(AttentionImpl): key: shape = [num_tokens, num_kv_heads, head_size] value: shape = [num_tokens, num_kv_heads, head_size] kv_cache: shape = - [num_blocks, 2, block_size, num_kv_heads, head_size] + [num_blocks, num_kv_heads, block_size, 2 * head_size] attn_metadata: Metadata for attention. Returns: shape = [num_tokens, num_heads * head_size] @@ -617,6 +633,7 @@ class TritonAttentionImpl(AttentionImpl): layer, ) + # KV cache arrives in logical (B, H, N, 2*hs) order. # Per-token-head quantized KV cache: handled by the core unified # kernel, which dequantizes per-(token, head) inline via constexpr # branches (INT8 / FP8) and dispatches to the packed INT4 kernel. @@ -627,7 +644,9 @@ class TritonAttentionImpl(AttentionImpl): q_descale = k_descale = v_descale = None # FP8 per-tensor / auto path (original flow). else: - key_cache, value_cache = kv_cache.unbind(1) + kv_cache = kv_cache.transpose(1, 2) + hs = self.head_size + key_cache, value_cache = kv_cache.split(hs, dim=-1) if ( is_quantized_kv_cache(self.kv_cache_dtype) and key_cache.dtype != self.fp8_dtype @@ -711,7 +730,8 @@ class TritonAttentionImpl(AttentionImpl): ) -> tuple[torch.Tensor, torch.Tensor]: """Per-token-head K/V cache views (ensures scale caches; FP8 retyped).""" self._ensure_scale_caches(kv_cache) - key_cache, value_cache = kv_cache.unbind(1) + padded_hs = kv_cache.shape[-1] // 2 + key_cache, value_cache = kv_cache.transpose(1, 2).split(padded_hs, dim=-1) if self._kv_quant_mode == KVQuantMode.FP8_PER_TOKEN_HEAD: key_cache = key_cache.view(self.fp8_dtype) value_cache = value_cache.view(self.fp8_dtype) @@ -778,13 +798,9 @@ class TritonAttentionImpl(AttentionImpl): return # Reshape the input keys and values and store them in the cache. if self._is_per_token_head_quant: - self._ensure_scale_caches(kv_cache) - key_cache, value_cache = kv_cache.unbind(1) + key_cache, value_cache = self._pth_key_value_caches(kv_cache) k_scale_cache = self._k_scale_cache v_scale_cache = self._v_scale_cache - if self._kv_quant_mode == KVQuantMode.FP8_PER_TOKEN_HEAD: - key_cache = key_cache.view(self.fp8_dtype) - value_cache = value_cache.view(self.fp8_dtype) triton_reshape_and_cache_flash_per_token_head_quant( key, value, @@ -797,7 +813,8 @@ class TritonAttentionImpl(AttentionImpl): ) return # For decoder and cross-attention, use KV cache as before. - key_cache, value_cache = kv_cache.unbind(1) + # (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs)) + key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1) if is_quantized_kv_cache(self.kv_cache_dtype): key_cache = key_cache.view(self.fp8_dtype) value_cache = value_cache.view(self.fp8_dtype) @@ -829,7 +846,8 @@ class TritonAttentionImpl(AttentionImpl): kv_cache: torch.Tensor, layer_slot_mapping: torch.Tensor, ): - key_cache, value_cache = kv_cache.unbind(1) + # (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs)) + key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1) flash_layout = True is_fp8_kv_cache = is_quantized_kv_cache(self.kv_cache_dtype) diff --git a/vllm/v1/attention/backends/triton_attn_diffkv.py b/vllm/v1/attention/backends/triton_attn_diffkv.py index 3420a0eba4..02ea8401cb 100644 --- a/vllm/v1/attention/backends/triton_attn_diffkv.py +++ b/vllm/v1/attention/backends/triton_attn_diffkv.py @@ -2,12 +2,9 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Triton attention backend with different K/V head dimensions (DiffKV). -The KV cache layout is identical to ``FlashAttentionDiffKVBackend`` — K -and V are packed along the last dim: - - [num_blocks, block_size, num_kv_heads, head_size_qk + head_size_v] - -so existing helpers (``triton_reshape_and_cache_flash_diffkv``) are reused. +The KV cache layout is identical to ``FlashAttentionDiffKVBackend``: K and V +are packed along the last dim in the logical shape +``[num_blocks, num_kv_heads, block_size, head_size_qk + head_size_v]``. """ from typing import ClassVar @@ -106,10 +103,12 @@ class TritonAttentionDiffKVBackend(TritonAttentionBackend): ) -> tuple[int, ...]: if block_size % 16 != 0: raise ValueError("Block size must be a multiple of 16.") + # Logical (blocks-first, head-major) layout: K and V (with their + # different head sizes) packed in the content dim. return ( num_blocks, - block_size, num_kv_heads, + block_size, head_size + TritonAttentionDiffKVBackend.head_size_v, ) @@ -117,19 +116,22 @@ class TritonAttentionDiffKVBackend(TritonAttentionBackend): def get_kv_cache_stride_order( include_num_layers_dimension: bool = False, ) -> tuple[int, ...]: + # `stride_order` indicates the permutation that gets us from + # `get_kv_cache_shape` (logical (B, H, N, C_k+C_v)) to the actual + # memory layout we want. cache_layout = get_kv_cache_layout() if cache_layout == "NHD" and include_num_layers_dimension: - # (num_blocks, num_layers, block_size, - # num_kv_heads, head_size + head_size_v) - return (1, 0, 2, 3, 4) + # (num_blocks, num_layers, block_size, num_kv_heads, C_k+C_v) + return (1, 0, 3, 2, 4) elif cache_layout == "NHD": - return (0, 1, 2, 3) - elif cache_layout == "HND" and include_num_layers_dimension: - # (num_blocks, num_kv_heads, num_layers, - # block_size, head_size + head_size_v) - return (1, 3, 0, 2, 4) - elif cache_layout == "HND": + # (num_blocks, block_size, num_kv_heads, C_k+C_v) return (0, 2, 1, 3) + elif cache_layout == "HND" and include_num_layers_dimension: + # (num_blocks, num_kv_heads, num_layers, block_size, C_k+C_v) + return (1, 2, 0, 3, 4) + elif cache_layout == "HND": + # (num_blocks, num_kv_heads, block_size, C_k+C_v) + return (0, 1, 2, 3) else: raise ValueError(f"Unknown cache layout format {cache_layout}.") @@ -175,13 +177,12 @@ class TritonAttentionDiffKVImpl(TritonAttentionImpl): kv_cache: torch.Tensor, slot_mapping: torch.Tensor, ) -> None: - # Cache is packed [..., head_size_qk + head_size_v]; the diffkv - # reshape kernel writes K to [..., :head_size_qk] and V to - # [..., head_size_qk:hqk+hv]. + # Cache is logical (B, H, N, C); the diffkv reshape kernel expects + # (B, N, H, C). triton_reshape_and_cache_flash_diffkv( key, value, - kv_cache, + kv_cache.transpose(1, 2), slot_mapping, self.kv_cache_dtype, layer._k_scale, @@ -210,7 +211,7 @@ class TritonAttentionDiffKVImpl(TritonAttentionImpl): query: [num_tokens, num_heads, head_size_qk] key: [num_tokens, num_kv_heads, head_size_qk] value: [num_tokens, num_kv_heads, head_size_v] - kv_cache: [num_blocks, block_size, num_kv_heads, + kv_cache: [num_blocks, num_kv_heads, block_size, head_size_qk + head_size_v] output: [num_tokens, num_heads, head_size_v] """ @@ -231,8 +232,8 @@ class TritonAttentionDiffKVImpl(TritonAttentionImpl): head_size_qk = self.head_size head_size_v = TritonAttentionDiffKVBackend.head_size_v - # Slice the packed cache into K / V views. Strides on dims 0/1/2 - # match the original cache; dim 3 stays contiguous (stride 1). + # Triton DiffKV kernels consume (B, N, H, D) cache views. + kv_cache = kv_cache.transpose(1, 2) key_cache = kv_cache[..., :head_size_qk] value_cache = kv_cache[..., head_size_qk : head_size_qk + head_size_v] diff --git a/vllm/v1/attention/backends/turboquant_attn.py b/vllm/v1/attention/backends/turboquant_attn.py index af4ab007a8..a274339afc 100644 --- a/vllm/v1/attention/backends/turboquant_attn.py +++ b/vllm/v1/attention/backends/turboquant_attn.py @@ -141,9 +141,10 @@ class TurboQuantAttentionBackend(AttentionBackend): Standard attention backends use (2, num_blocks, block_size, num_kv_heads, head_dim) with a leading 2 to separate K and V. TurboQuant packs K+V - into a single interleaved slot per head per position, so the cache is: + into a single interleaved slot per head per position. The logical + (blocks-first, head-major) shape is: - (num_blocks, block_size, num_kv_heads, slot_size_aligned) + (num_blocks, num_kv_heads, block_size, slot_size_aligned) Each slot = [key_packed | value_packed | padding]. This is safe because TQ has its own get_kv_cache_shape override and @@ -159,7 +160,7 @@ class TurboQuantAttentionBackend(AttentionBackend): ) tq_config = TurboQuantConfig.from_cache_dtype(cache_dtype_str, head_size) - return (num_blocks, block_size, num_kv_heads, tq_config.slot_size_aligned) + return (num_blocks, num_kv_heads, block_size, tq_config.slot_size_aligned) @classmethod def supports_kv_cache_dtype(cls, kv_cache_dtype: CacheDType | None) -> bool: @@ -422,6 +423,8 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]): k = key[:N].view(N, self.num_kv_heads, self.head_size) v = value[:N].view(N, self.num_kv_heads, self.head_size) + # (B, H, N, C) -> (B, N, H, C) for TQ kernels + kv_cache = kv_cache.transpose(1, 2) self._store_kv(k, v, kv_cache, slot_mapping, layer) def forward( @@ -449,6 +452,9 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]): if attn_metadata is None: return output.fill_(0) + # (B, H, N, C) -> (B, N, H, C) for TQ kernels + kv_cache = kv_cache.transpose(1, 2) + # Slice to actual tokens N = attn_metadata.num_actual_tokens if N <= 0: diff --git a/vllm/v1/attention/ops/triton_reshape_and_cache_flash.py b/vllm/v1/attention/ops/triton_reshape_and_cache_flash.py index 0f0022c2cb..882b6787ae 100644 --- a/vllm/v1/attention/ops/triton_reshape_and_cache_flash.py +++ b/vllm/v1/attention/ops/triton_reshape_and_cache_flash.py @@ -473,6 +473,7 @@ def reshape_and_cache_kernel_flash_diffkv( value_stride: tl.int64, block_stride: tl.int64, page_stride: tl.int64, + head_stride: tl.int64, num_heads: tl.constexpr, head_size_k: tl.constexpr, head_size_v: tl.constexpr, @@ -498,9 +499,7 @@ def reshape_and_cache_kernel_flash_diffkv( src_value_idx = token_idx * value_stride + tile_i * head_size_v tgt_idx = ( - block_idx * block_stride - + block_offset * page_stride - + tile_i * (head_size_k + head_size_v) + block_idx * block_stride + block_offset * page_stride + tile_i * head_stride ) # [TILE_SIZE] @@ -542,7 +541,7 @@ def reshape_and_cache_kernel_flash_diffkv( def triton_reshape_and_cache_flash_diffkv( key: torch.Tensor, # [num_tokens, num_heads, head_size] value: torch.Tensor, # [num_tokens, num_heads, head_size_v] - # [num_blocks, block_size, num_heads, head_size + head_size_v] + # Strided [num_blocks, block_size, num_heads, head_size + head_size_v]. kv_cache: torch.Tensor, slot_mapping: torch.Tensor, # [num_tokens] kv_cache_dtype: str, # "auto", "fp8" @@ -558,6 +557,7 @@ def triton_reshape_and_cache_flash_diffkv( v_stride = value.stride()[0] block_stride = kv_cache.stride()[0] page_stride = kv_cache.stride()[1] + head_stride = kv_cache.stride()[2] kv_cache_torch_dtype = ( current_platform.fp8_dtype() @@ -599,6 +599,7 @@ def triton_reshape_and_cache_flash_diffkv( value_stride=v_stride, block_stride=block_stride, page_stride=page_stride, + head_stride=head_stride, num_heads=num_heads, head_size_k=head_size_k, head_size_v=head_size_v, diff --git a/vllm/v1/attention/ops/triton_turboquant_store.py b/vllm/v1/attention/ops/triton_turboquant_store.py index d5437d7314..e680f6883a 100644 --- a/vllm/v1/attention/ops/triton_turboquant_store.py +++ b/vllm/v1/attention/ops/triton_turboquant_store.py @@ -389,7 +389,7 @@ def triton_turboquant_store( _tq_fused_store_fp8[grid]( k_flat, v_flat, - kv_cache.view(-1), + kv_cache, slot_mapping, stride_cache_block=stride_block, stride_cache_pos=stride_pos, @@ -425,7 +425,7 @@ def triton_turboquant_store( norms.squeeze(1), v_flat, midpoints, - kv_cache.view(-1), + kv_cache, slot_mapping, stride_cache_block=stride_block, stride_cache_pos=stride_pos,