[2/N][KV-Cache Layout Refactor] Pack K/V into the content dim across attention backends (#44455)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
co-authored by
Claude
OpenAI Codex
Matthew Bonanni
Nicolò Lucchesi
parent
76fedaa2a5
commit
51878e5b6e
@@ -288,16 +288,15 @@ __global__ void fusedMiniMaxM3QNormRopeKVInsertKernel(
|
||||
int64_t const* __restrict__ positions, // [N] i64
|
||||
int64_t const* __restrict__ slot_mapping, // main K/V slots or nullptr
|
||||
int64_t const* __restrict__ index_slot_mapping, // index K slots/nullptr
|
||||
cache_t* __restrict__ kv_cache, // [nb,2,bs,nkv,128] or nullptr
|
||||
cache_t* __restrict__ kv_cache, // [nb,nkv,bs,2*128] or nullptr
|
||||
out_idx_t* __restrict__ index_cache, // [nb*bs, 128]; scalar_t or e4m3 byte
|
||||
float const eps, int const rotary_dim, int const num_tokens, int const nq,
|
||||
int const nkv, int const niq, int const block_size,
|
||||
// kv_cache strides (in elements) for logical shape [nb, 2, bs, nkv, 128].
|
||||
// The head_dim (last) dim is always innermost-contiguous (stride 1), so the
|
||||
// NHD/HND layout choice is fully captured by these four strides: NHD keeps
|
||||
// s_token < s_head, HND swaps them. dim_base addresses head_dim directly.
|
||||
int64_t const kv_s_block, int64_t const kv_s_kv, int64_t const kv_s_token,
|
||||
int64_t const kv_s_head) {
|
||||
// kv_cache strides (in elements) for logical shape [nb, nkv, bs, 2*128].
|
||||
// The content (last) dim is always innermost-contiguous (stride 1), so the
|
||||
// NHD/HND layout choice is captured by the head/token strides.
|
||||
int64_t const kv_s_block, int64_t const kv_s_head, int64_t const kv_s_token,
|
||||
int64_t const kv_s_dim) {
|
||||
#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)
|
||||
// _typeConvert<BFloat16> is unavailable on pre-Ampere; the M3 kernel only
|
||||
// runs with bf16/fp16 inputs in practice. Discard the bf16 body there.
|
||||
@@ -438,16 +437,16 @@ __global__ void fusedMiniMaxM3QNormRopeKVInsertKernel(
|
||||
storeElems<scalar_t>(index_cache + sm * kHeadDim + dim_base, elems);
|
||||
}
|
||||
} else if (isK || isV) {
|
||||
// kv_cache logical shape [num_blocks, 2, block_size, nkv, head_dim].
|
||||
// kv_cache logical shape [num_blocks, nkv, block_size, 2*head_dim].
|
||||
// Paging is logical (block = sm/block_size, token = sm%block_size);
|
||||
// the physical NHD/HND layout is honoured via the passed strides.
|
||||
int64_t const b = sm / block_size;
|
||||
int64_t const t = sm % block_size;
|
||||
int const kv = isK ? 0 : 1;
|
||||
int64_t const off =
|
||||
b * kv_s_block + kv * kv_s_kv + t * kv_s_token + head * kv_s_head;
|
||||
storeCacheElems<scalar_t, cache_t, kv_dt>(kv_cache + off + dim_base,
|
||||
elems);
|
||||
int64_t const off = b * kv_s_block + head * kv_s_head +
|
||||
t * kv_s_token +
|
||||
(kv * kHeadDim + dim_base) * kv_s_dim;
|
||||
storeCacheElems<scalar_t, cache_t, kv_dt>(kv_cache + off, elems);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -474,8 +473,8 @@ void launchFusedMiniMaxM3(
|
||||
int64_t const* index_slot_mapping, cache_t* kv_cache, void* index_cache,
|
||||
float const eps, int const rotary_dim, int const num_tokens, int const nq,
|
||||
int const nkv, int const niq, int const block_size,
|
||||
int64_t const kv_s_block, int64_t const kv_s_kv, int64_t const kv_s_token,
|
||||
int64_t const kv_s_head, bool const has_index, bool const insert_kv,
|
||||
int64_t const kv_s_block, int64_t const kv_s_head, int64_t const kv_s_token,
|
||||
int64_t const kv_s_dim, bool const has_index, bool const insert_kv,
|
||||
bool const fp8_idx, cudaStream_t stream) {
|
||||
// Index outputs are scalar_t (bf16) or e4m3 bytes (uint8_t); reinterpret the
|
||||
// void* pointers per instantiation in the LAUNCH macro.
|
||||
@@ -517,20 +516,20 @@ void launchFusedMiniMaxM3(
|
||||
iq_norm_w, ik_norm_w, cos_sin_cache, positions, slot_mapping, \
|
||||
index_slot_mapping, kv_cache, reinterpret_cast<OUT_T*>(index_cache), \
|
||||
eps, rotary_dim, num_tokens, nq, nkv, niq, block_size, kv_s_block, \
|
||||
kv_s_kv, kv_s_token, kv_s_head)
|
||||
kv_s_head, kv_s_token, kv_s_dim)
|
||||
#else
|
||||
// ROCm: standard kernel launch syntax (no PDL/stream serialization).
|
||||
// clang-format off
|
||||
#define LAUNCH(IS_SPARSE, INSERT, FP8, OUT_T) \
|
||||
fusedMiniMaxM3QNormRopeKVInsertKernel<scalar_t, cache_t, kv_dt, OUT_T, \
|
||||
fusedMiniMaxM3QNormRopeKVInsertKernel<scalar_t, cache_t, kv_dt, OUT_T, \
|
||||
IS_SPARSE, INSERT, FP8> \
|
||||
<<<grid, kBlockSize, 0, stream>>>( \
|
||||
qkv, q_out, reinterpret_cast<OUT_T*>(index_q_out), q_norm_w, \
|
||||
k_norm_w, iq_norm_w, ik_norm_w, cos_sin_cache, positions, \
|
||||
slot_mapping, index_slot_mapping, kv_cache, \
|
||||
reinterpret_cast<OUT_T*>(index_cache), eps, rotary_dim, \
|
||||
num_tokens, nq, nkv, niq, block_size, kv_s_block, kv_s_kv, \
|
||||
kv_s_token, kv_s_head)
|
||||
num_tokens, nq, nkv, niq, block_size, kv_s_block, kv_s_head, \
|
||||
kv_s_token, kv_s_dim)
|
||||
// clang-format on
|
||||
#endif
|
||||
|
||||
@@ -584,8 +583,8 @@ void launchFusedMiniMaxM3(
|
||||
? reinterpret_cast<void*>(index_cache->data_ptr()) \
|
||||
: nullptr, \
|
||||
static_cast<float>(eps), static_cast<int>(rotary_dim), num_tokens, nq, \
|
||||
nkv, niq, static_cast<int>(block_size), kv_s_block, kv_s_kv, kv_s_token, \
|
||||
kv_s_head, has_index, insert_kv, fp8_idx, stream)
|
||||
nkv, niq, static_cast<int>(block_size), kv_s_block, kv_s_head, \
|
||||
kv_s_token, kv_s_dim, has_index, insert_kv, fp8_idx, stream)
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
// Torch op wrapper
|
||||
@@ -602,7 +601,7 @@ void fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
int64_t num_index_heads, // niq; 0 => dense
|
||||
std::optional<torch::stable::Tensor> slot_mapping, // [N] i64
|
||||
std::optional<torch::stable::Tensor> index_slot_mapping, // [N] i64
|
||||
std::optional<torch::stable::Tensor> kv_cache, // [nb,2,bs,nkv,128]
|
||||
std::optional<torch::stable::Tensor> kv_cache, // [nb,nkv,bs,2*128]
|
||||
std::optional<torch::stable::Tensor> index_cache, // [nb,bs,128]
|
||||
int64_t block_size,
|
||||
std::optional<torch::stable::Tensor> q_out, // [N, nq*128] contiguous
|
||||
@@ -673,11 +672,11 @@ void fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
index_k_norm_weight->numel() == kHeadDim,
|
||||
"index norm weights must have 128 elements");
|
||||
}
|
||||
// kv_cache strides (logical shape [nb, 2, bs, nkv, head_dim]). Read straight
|
||||
// kv_cache strides (logical shape [nb, nkv, bs, 2*head_dim]). Read straight
|
||||
// off the tensor so the kernel honours whatever physical layout the attention
|
||||
// backend allocated (NHD: stride order (0,1,2,3,4); HND: (0,1,3,2,4)). No new
|
||||
// backend allocated (NHD: stride order (0,2,1,3); HND: (0,1,2,3)). No new
|
||||
// op argument is needed -- the strides ride along with the tensor itself.
|
||||
int64_t kv_s_block = 0, kv_s_kv = 0, kv_s_token = 0, kv_s_head = 0;
|
||||
int64_t kv_s_block = 0, kv_s_head = 0, kv_s_token = 0, kv_s_dim = 0;
|
||||
torch::stable::Tensor const* effective_index_slot_mapping = nullptr;
|
||||
if (insert_kv) {
|
||||
STD_TORCH_CHECK(
|
||||
@@ -707,13 +706,13 @@ void fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
index_cache->scalar_type() ==
|
||||
torch::headeronly::ScalarType::Float8_e4m3fn),
|
||||
"insert mode requires index_cache matching qkv dtype or fp8 e4m3");
|
||||
STD_TORCH_CHECK(kv_cache->dim() == 5 && kv_cache->stride(4) == 1,
|
||||
"kv_cache must be [nb,2,bs,nkv,head_dim] with contiguous "
|
||||
"head_dim (stride(4)==1)");
|
||||
STD_TORCH_CHECK(kv_cache->dim() == 4 && kv_cache->stride(3) == 1,
|
||||
"kv_cache must be [nb,nkv,bs,2*head_dim] with contiguous "
|
||||
"content dim (stride(3)==1)");
|
||||
kv_s_block = kv_cache->stride(0);
|
||||
kv_s_kv = kv_cache->stride(1);
|
||||
kv_s_head = kv_cache->stride(1);
|
||||
kv_s_token = kv_cache->stride(2);
|
||||
kv_s_head = kv_cache->stride(3);
|
||||
kv_s_dim = kv_cache->stride(3);
|
||||
effective_index_slot_mapping = index_slot_mapping.has_value()
|
||||
? &index_slot_mapping.value()
|
||||
: &slot_mapping.value();
|
||||
|
||||
@@ -111,7 +111,11 @@ class AttentionQuantPatternModel(torch.nn.Module):
|
||||
# Fetch the attention backend and kv cache shape and stride order
|
||||
attn_backend = self.attn.attn_backend
|
||||
kv_cache_shape = attn_backend.get_kv_cache_shape(
|
||||
num_blocks, self.block_size, self.num_kv_heads, self.head_size
|
||||
num_blocks,
|
||||
self.block_size,
|
||||
self.num_kv_heads,
|
||||
self.head_size,
|
||||
cache_dtype_str=self.attn.kv_cache_dtype,
|
||||
)
|
||||
try:
|
||||
kv_cache_stride_order = attn_backend.get_kv_cache_stride_order()
|
||||
@@ -125,11 +129,10 @@ class AttentionQuantPatternModel(torch.nn.Module):
|
||||
|
||||
# Create dummy KV cache
|
||||
raw_tensor = torch.zeros(
|
||||
2 * num_blocks * self.block_size * self.num_kv_heads * self.head_size,
|
||||
kv_cache_shape,
|
||||
dtype=self.attn.kv_cache_torch_dtype,
|
||||
device=self.device,
|
||||
)
|
||||
raw_tensor = raw_tensor.view(kv_cache_shape)
|
||||
kv_cache = raw_tensor.permute(*inv_order)
|
||||
|
||||
self.attn.kv_cache = kv_cache
|
||||
|
||||
@@ -10,7 +10,7 @@ from tests.kernels.utils import DEFAULT_OPCHECK_TEST_UTILS, opcheck
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import scaled_dequantize
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import nvfp4_kv_cache_split_views, set_random_seed
|
||||
from vllm.utils.torch_utils import nvfp4_split_data_scale, set_random_seed
|
||||
|
||||
COPYING_DIRECTION = [("cuda", "cpu"), ("cuda", "cuda"), ("cpu", "cuda")]
|
||||
DTYPES = [torch.bfloat16, torch.float]
|
||||
@@ -255,10 +255,8 @@ def test_reshape_and_cache_flash(
|
||||
nvfp4_key_data = None
|
||||
nvfp4_value_data = None
|
||||
if kv_cache_dtype == "nvfp4":
|
||||
(nvfp4_key_data,), (key_scale_cache,) = nvfp4_kv_cache_split_views(key_cache)
|
||||
(nvfp4_value_data,), (value_scale_cache,) = nvfp4_kv_cache_split_views(
|
||||
value_cache
|
||||
)
|
||||
nvfp4_key_data, key_scale_cache = nvfp4_split_data_scale(key_cache)
|
||||
nvfp4_value_data, value_scale_cache = nvfp4_split_data_scale(value_cache)
|
||||
|
||||
if kv_cache_dtype == "nvfp4":
|
||||
# Global scale = amax / 448 (per-tensor)
|
||||
|
||||
@@ -13,7 +13,7 @@ from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.utils.torch_utils import (
|
||||
nvfp4_kv_cache_full_dim,
|
||||
nvfp4_kv_cache_split_views,
|
||||
nvfp4_split_data_scale,
|
||||
set_random_seed,
|
||||
)
|
||||
|
||||
@@ -74,17 +74,18 @@ def make_nvfp4_kv_cache(
|
||||
kv_scale_val, dtype=torch.float32, device=kv_bf16_hnd.device
|
||||
)
|
||||
|
||||
# Allocate in HND physical order, permute to NHD logical order.
|
||||
# hnd_order swaps dims 2↔3; it is its own inverse.
|
||||
# layout: (B, 2*H, N, full_dim)
|
||||
# where K heads occupy the first H heads and V heads occupy the second H heads.
|
||||
full_dim = nvfp4_kv_cache_full_dim(head_size)
|
||||
hnd_order = (0, 1, 3, 2, 4)
|
||||
kv_cache = torch.zeros(
|
||||
(num_blocks, 2, num_kv_heads, block_size, full_dim),
|
||||
kv_cache_hnd = torch.zeros(
|
||||
(num_blocks, 2 * num_kv_heads, block_size, full_dim),
|
||||
dtype=torch.uint8,
|
||||
device=kv_bf16_hnd.device,
|
||||
).permute(*hnd_order)
|
||||
)
|
||||
kv_cache_nhd = kv_cache_hnd.permute(0, 2, 1, 3)
|
||||
k_view_nhd, v_view_nhd = kv_cache_nhd.split(num_kv_heads, dim=-2)
|
||||
|
||||
# Flatten NHD [N, T, H, D] → token tensors [N*T, H, D] for the kernel.
|
||||
# Flatten input KV → token tensors [B*N, H, head_size] for the kernel.
|
||||
num_tokens = num_blocks * block_size
|
||||
k_tokens = (
|
||||
kv_bf16_hnd[:, 0]
|
||||
@@ -98,22 +99,21 @@ def make_nvfp4_kv_cache(
|
||||
)
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.long, device=kv_bf16_hnd.device)
|
||||
|
||||
# reshape_and_cache_flash: kernel receives kv_cache[:, 0] and [:, 1]
|
||||
# (full K/V buffers containing both data and scale).
|
||||
torch.ops._C_cache_ops.reshape_and_cache_flash(
|
||||
k_tokens,
|
||||
v_tokens,
|
||||
kv_cache[:, 0],
|
||||
kv_cache[:, 1],
|
||||
k_view_nhd,
|
||||
v_view_nhd,
|
||||
slot_mapping,
|
||||
"nvfp4",
|
||||
kv_scale_tensor,
|
||||
kv_scale_tensor,
|
||||
)
|
||||
|
||||
# Split in HND order for trtllm kernel (expects HND numTokensPerPage).
|
||||
kv_cache_hnd = kv_cache.permute(*hnd_order)
|
||||
(k_data, v_data), (k_scales, v_scales) = nvfp4_kv_cache_split_views(kv_cache_hnd)
|
||||
# Split into data/scale views in HNC order for trtllm kernel.
|
||||
k_cache_hnc, v_cache_hnc = kv_cache_hnd.split(num_kv_heads, dim=1)
|
||||
k_data, k_scales = nvfp4_split_data_scale(k_cache_hnc)
|
||||
v_data, v_scales = nvfp4_split_data_scale(v_cache_hnc)
|
||||
|
||||
# Dequantize for the FA2 reference baseline.
|
||||
ref_k = dequant_nvfp4_kv_cache(
|
||||
|
||||
@@ -662,8 +662,9 @@ def _reference_sparse_attn(
|
||||
positions = torch.arange(seq_len, device="cuda")
|
||||
pages = block_table[req_id, positions // BLOCK_SIZE]
|
||||
rows = positions % BLOCK_SIZE
|
||||
k_req = kv_cache[pages, 0, rows]
|
||||
v_req = kv_cache[pages, 1, rows].float()
|
||||
kv_req = kv_cache[pages, :, rows]
|
||||
k_req = kv_req[..., :HEAD_DIM]
|
||||
v_req = kv_req[..., HEAD_DIM:].float()
|
||||
|
||||
q_pos = prefix_len + torch.arange(q_len, device="cuda")
|
||||
key_blocks = positions // BLOCK_SIZE
|
||||
@@ -791,15 +792,15 @@ def test_main_backend_layout_contract():
|
||||
flash_attn-style stride order for each layout."""
|
||||
nb, bs, h, d = 7, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
|
||||
logical = MiniMaxM3SparseBackend.get_kv_cache_shape(nb, bs, h, d)
|
||||
assert logical == (nb, 2, bs, h, d)
|
||||
# The old HND-ordered shape is no longer the logical shape.
|
||||
assert logical != (nb, 2, h, bs, d)
|
||||
assert logical == (nb, h, bs, 2 * d)
|
||||
# The old separate K/V-axis shape is no longer the logical shape.
|
||||
assert logical != (nb, 2, bs, h, d)
|
||||
|
||||
try:
|
||||
set_kv_cache_layout("HND")
|
||||
assert MiniMaxM3SparseBackend.get_kv_cache_stride_order() == (0, 1, 3, 2, 4)
|
||||
assert MiniMaxM3SparseBackend.get_kv_cache_stride_order() == (0, 1, 2, 3)
|
||||
set_kv_cache_layout("NHD")
|
||||
assert MiniMaxM3SparseBackend.get_kv_cache_stride_order() == (0, 1, 2, 3, 4)
|
||||
assert MiniMaxM3SparseBackend.get_kv_cache_stride_order() == (0, 2, 1, 3)
|
||||
finally:
|
||||
set_kv_cache_layout(None)
|
||||
|
||||
@@ -829,7 +830,7 @@ def test_main_backend_unknown_layout_raises(monkeypatch):
|
||||
|
||||
|
||||
def test_indexer_backend_stride_order_is_identity():
|
||||
"""The 3-dim indexer cache must not inherit the parent's 5-element stride
|
||||
"""The 3-dim indexer cache must not inherit the parent's 4-element stride
|
||||
order; it overrides to the 3-element identity so the allocator keeps the
|
||||
contiguous layout."""
|
||||
assert MiniMaxM3IndexerBackend.get_kv_cache_stride_order() == (0, 1, 2)
|
||||
@@ -848,9 +849,9 @@ def test_indexer_backend_stride_order_is_identity():
|
||||
assert _stride_order_for(MiniMaxM3IndexerBackend, len(indexer_shape)) == (0, 1, 2)
|
||||
|
||||
|
||||
def test_hnd_allocation_is_byte_identical_to_transpose():
|
||||
"""Under HND the backend-visible logical view is byte-identical to the
|
||||
pre-change allocate-HND-then-transpose(2, 3) workaround."""
|
||||
def test_hnd_allocation_is_packed_head_major():
|
||||
"""Under HND the backend-visible logical view is the packed head-major
|
||||
physical allocation."""
|
||||
nb, bs, h, d = 4, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
|
||||
logical = MiniMaxM3SparseBackend.get_kv_cache_shape(nb, bs, h, d)
|
||||
try:
|
||||
@@ -860,21 +861,22 @@ def test_hnd_allocation_is_byte_identical_to_transpose():
|
||||
set_kv_cache_layout(None)
|
||||
|
||||
physical_shape = tuple(logical[i] for i in stride_order)
|
||||
# The physical (permuted) shape equals the old hardcoded HND shape.
|
||||
assert physical_shape == (nb, 2, h, bs, d)
|
||||
assert physical_shape == (nb, h, bs, 2 * d)
|
||||
|
||||
inv_order = [stride_order.index(i) for i in range(len(stride_order))]
|
||||
raw = torch.empty(physical_shape, device="cuda", dtype=DTYPE)
|
||||
view = raw.permute(*inv_order)
|
||||
expected = raw.view((nb, 2, h, bs, d)).transpose(2, 3)
|
||||
expected = raw.view((nb, h, bs, 2 * d))
|
||||
|
||||
assert view.shape == expected.shape
|
||||
assert view.stride() == expected.stride()
|
||||
assert view.storage_offset() == expected.storage_offset()
|
||||
|
||||
# Negative: the identity (wrong) stride order under HND does not reproduce
|
||||
# the transpose view.
|
||||
wrong_view = raw.view(logical)
|
||||
# Negative: the NHD stride order under HND does not reproduce the
|
||||
# head-major view.
|
||||
wrong_order = (0, 2, 1, 3)
|
||||
wrong_inv = [wrong_order.index(i) for i in range(len(wrong_order))]
|
||||
wrong_view = raw.view(tuple(logical[i] for i in wrong_order)).permute(*wrong_inv)
|
||||
assert wrong_view.stride() != expected.stride()
|
||||
|
||||
|
||||
@@ -1037,14 +1039,18 @@ def test_decode_wrong_layout_breaks_parity():
|
||||
seq_lens_list = (130, 257)
|
||||
q, block_table, seq_lens, topk_idx, num_pages = _build_decode_inputs(seq_lens_list)
|
||||
|
||||
# Physical HND storage [blocks, 2, heads, block, dim].
|
||||
# Physical HND storage [blocks, heads, block, packed_kv_dim].
|
||||
phys = torch.randn(
|
||||
(num_pages, 2, NUM_KV_HEADS, BLOCK_SIZE, HEAD_DIM), device="cuda", dtype=DTYPE
|
||||
(num_pages, NUM_KV_HEADS, BLOCK_SIZE, 2 * HEAD_DIM),
|
||||
device="cuda",
|
||||
dtype=DTYPE,
|
||||
)
|
||||
# Correct logical packed-HND view vs. the same bytes mislabeled as NHD
|
||||
# physical storage and then exposed as a logical cache.
|
||||
correct = phys
|
||||
wrong = phys.reshape(num_pages, BLOCK_SIZE, NUM_KV_HEADS, 2 * HEAD_DIM).permute(
|
||||
0, 2, 1, 3
|
||||
)
|
||||
# Correct logical-NHD view (strided) vs. the same bytes mislabeled as a
|
||||
# contiguous-NHD cache — same shape, different content mapping.
|
||||
correct = phys.permute(0, 1, 3, 2, 4)
|
||||
wrong = phys.reshape(num_pages, 2, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM)
|
||||
|
||||
q_lens_t = torch.ones(len(seq_lens_list), device="cuda", dtype=torch.int32)
|
||||
prefix_lens = seq_lens - q_lens_t
|
||||
@@ -1072,9 +1078,9 @@ def _make_attn_group(backend, spec):
|
||||
def test_main_cache_byte_identical_through_production_allocator():
|
||||
"""AC-2: drive the real allocator (`_reshape_kv_cache`) for the M3 main
|
||||
`FullAttentionSpec` under HND and assert the backend-visible view has the
|
||||
same shape, stride, and storage offset as the pre-change
|
||||
allocate-HND-then-transpose path; the indexer `MLAAttentionSpec` allocates
|
||||
through the same path to its 3-dim shape."""
|
||||
same shape, stride, and storage offset as the packed-HND allocation; the
|
||||
indexer `MLAAttentionSpec` allocates through the same path to its 3-dim
|
||||
shape."""
|
||||
nb = 4
|
||||
spec = FullAttentionSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
@@ -1092,8 +1098,7 @@ def test_main_cache_byte_identical_through_production_allocator():
|
||||
set_kv_cache_layout(None)
|
||||
view = kv_caches["main"]
|
||||
|
||||
oracle = raw.view(DTYPE).view((nb, 2, NUM_KV_HEADS, BLOCK_SIZE, HEAD_DIM))
|
||||
oracle = oracle.transpose(2, 3)
|
||||
oracle = raw.view(DTYPE).view((nb, NUM_KV_HEADS, BLOCK_SIZE, 2 * HEAD_DIM))
|
||||
assert tuple(view.shape) == tuple(oracle.shape)
|
||||
assert view.stride() == oracle.stride()
|
||||
assert view.storage_offset() == oracle.storage_offset()
|
||||
@@ -1119,13 +1124,13 @@ def test_main_cache_byte_identical_through_production_allocator():
|
||||
|
||||
|
||||
def test_indexer_inherited_stride_order_trips_allocator_assert():
|
||||
"""AC-4 negative: without the indexer override, the inherited 5-element
|
||||
"""AC-4 negative: without the indexer override, the inherited 4-element
|
||||
stride order trips the allocator's `len(stride_order) == len(shape)` assert
|
||||
for the 3-dim indexer shape; the `AssertionError` is NOT swallowed by the
|
||||
allocator's `(AttributeError, NotImplementedError)` fallback."""
|
||||
|
||||
class _BrokenIndexerBackend(MiniMaxM3IndexerBackend):
|
||||
# Simulate inheriting the parent's 5-element stride order.
|
||||
# Simulate inheriting the parent's 4-element stride order.
|
||||
get_kv_cache_stride_order = staticmethod(
|
||||
MiniMaxM3SparseBackend.get_kv_cache_stride_order
|
||||
)
|
||||
@@ -1192,10 +1197,8 @@ def test_padded_main_cache_is_flagged():
|
||||
@pytest.mark.parametrize("kv_layout", ["NHD", "HND"], indirect=True)
|
||||
def test_reshape_and_cache_flash_write_persists(kv_layout: str):
|
||||
"""AC-5 write path: the `reshape_and_cache_flash` write site now consumes
|
||||
`self.kv_cache.unbind(1)` directly. Writing through those views must persist
|
||||
into the bound storage (read back through an independent logical view) under
|
||||
both layouts — a `.contiguous()` copy of the unbind slice would leave the
|
||||
bound storage unchanged."""
|
||||
packed-content K/V split views. Writing through those views must persist
|
||||
into the bound storage under both layouts."""
|
||||
torch.manual_seed(0)
|
||||
num_pages = 4
|
||||
kv_cache = _allocate_main_kv_via_contract(num_pages)
|
||||
@@ -1203,7 +1206,7 @@ def test_reshape_and_cache_flash_write_persists(kv_layout: str):
|
||||
kv_cache.zero_()
|
||||
|
||||
# Exactly the production write-site code under test.
|
||||
key_cache, value_cache = kv_cache.unbind(1)
|
||||
key_cache, value_cache = kv_cache.transpose(1, 2).split(HEAD_DIM, dim=-1)
|
||||
|
||||
num_tokens = 12
|
||||
slot_mapping = torch.randperm(num_pages * BLOCK_SIZE, device="cuda")[
|
||||
@@ -1222,5 +1225,5 @@ def test_reshape_and_cache_flash_write_persists(kv_layout: str):
|
||||
for t in range(num_tokens):
|
||||
slot = int(slot_mapping[t].item())
|
||||
blk, intra = divmod(slot, BLOCK_SIZE)
|
||||
torch.testing.assert_close(kv_cache[blk, 0, intra], key[t])
|
||||
torch.testing.assert_close(kv_cache[blk, 1, intra], value[t])
|
||||
torch.testing.assert_close(kv_cache[blk, :, intra, :HEAD_DIM], key[t])
|
||||
torch.testing.assert_close(kv_cache[blk, :, intra, HEAD_DIM:], value[t])
|
||||
|
||||
@@ -140,12 +140,10 @@ def create_and_prepopulate_kv_cache(
|
||||
block_table = common_attn_metadata.block_table_tensor
|
||||
slot_mapping = common_attn_metadata.slot_mapping
|
||||
|
||||
# Create KV cache and populate in (2, num_blocks, ...) layout for easy
|
||||
# flat indexing, then transpose to (num_blocks, 2, ...) layout.
|
||||
kv_cache = torch.zeros(
|
||||
2, num_blocks, block_size, num_kv_heads, head_size, dtype=dtype, device=device
|
||||
num_blocks, block_size, num_kv_heads, 2 * head_size, dtype=dtype, device=device
|
||||
)
|
||||
kv_cache_flat = kv_cache.view(2, -1, num_kv_heads, head_size)
|
||||
kv_cache_flat = kv_cache.view(-1, num_kv_heads, 2 * head_size)
|
||||
|
||||
# Populate the cache with the context tokens
|
||||
# Start from block_id=1 since block_id=0 is considered the null block
|
||||
@@ -154,15 +152,12 @@ def create_and_prepopulate_kv_cache(
|
||||
k_context, v_context = k_contexts[i], v_contexts[i]
|
||||
start = start_block_idx * block_size
|
||||
end = start + k_context.shape[0]
|
||||
kv_cache_flat[0, start:end, ...] = k_context
|
||||
kv_cache_flat[1, start:end, ...] = v_context
|
||||
kv_cache_flat[start:end, :, :head_size] = k_context
|
||||
kv_cache_flat[start:end, :, head_size:] = v_context
|
||||
|
||||
# Stay block aligned and allocate enough blocks for the new tokens
|
||||
start_block_idx += cdiv(int(seq_lens[i]), block_size)
|
||||
|
||||
# Transpose to (num_blocks, 2, ...) layout
|
||||
kv_cache = kv_cache.transpose(0, 1).contiguous()
|
||||
|
||||
blocks_end = start_block_idx
|
||||
|
||||
# Permute the context blocks (excluding block 0 which is null)
|
||||
@@ -199,7 +194,8 @@ def create_and_prepopulate_kv_cache(
|
||||
i, block_indices
|
||||
] * block_size + token_inter_block_offsets.to(device)
|
||||
|
||||
return kv_cache
|
||||
# Transpose to logical (num_blocks, num_kv_heads, block_size, 2*hs)
|
||||
return kv_cache.transpose(1, 2).contiguous()
|
||||
|
||||
|
||||
class MockAttentionLayer:
|
||||
@@ -496,9 +492,6 @@ def _test_backend_correctness(
|
||||
set_kv_cache_layout("HND")
|
||||
reset_kv_cache_layout = True
|
||||
|
||||
# Apply stride order like runtime does in
|
||||
# _reshape_kv_cache (attn_utils.py:182-210): permute to physical
|
||||
# layout, make contiguous, then permute to logical layout.
|
||||
kv_cache_for_backend = kv_cache
|
||||
if backend_cls is not None:
|
||||
try:
|
||||
@@ -506,6 +499,9 @@ def _test_backend_correctness(
|
||||
except (AttributeError, NotImplementedError):
|
||||
stride_order = tuple(range(kv_cache.ndim))
|
||||
if stride_order != tuple(range(kv_cache.ndim)):
|
||||
# Apply stride order like runtime does in
|
||||
# _reshape_kv_cache (attn_utils.py:182-210): permute to physical
|
||||
# layout, make contiguous, then permute to logical layout.
|
||||
inv_order = [stride_order.index(i) for i in range(len(stride_order))]
|
||||
kv_cache_for_backend = (
|
||||
kv_cache.permute(*stride_order).contiguous().permute(*inv_order)
|
||||
|
||||
@@ -97,13 +97,13 @@ def _create_hnd_kv_cache(
|
||||
device,
|
||||
num_blocks,
|
||||
common_attn_metadata,
|
||||
kv_in_head_dim=False,
|
||||
):
|
||||
"""Create and populate a KV cache with HND-compatible strides.
|
||||
"""Create and populate a packed KV cache with HND-compatible strides.
|
||||
|
||||
The returned tensor has logical shape
|
||||
(num_blocks, 2, block_size, num_kv_heads, head_size) but is physically
|
||||
laid out as (num_blocks, 2, num_kv_heads, block_size, head_size) so that
|
||||
``kv_cache.permute(0, 1, 3, 2, 4)`` yields a contiguous HND view.
|
||||
When kv_in_head_dim=False (default), returns (B, H, N, 2*hs) with K/V
|
||||
packed in the content dim. When kv_in_head_dim=True, returns
|
||||
(B, 2*H, N, hs) with K/V as separate head groups.
|
||||
"""
|
||||
seq_lens = common_attn_metadata.seq_lens.cpu()
|
||||
query_lens = (
|
||||
@@ -114,26 +114,34 @@ def _create_hnd_kv_cache(
|
||||
slot_mapping = common_attn_metadata.slot_mapping
|
||||
batch_size = len(k_contexts)
|
||||
|
||||
# Build cache in (2, num_blocks, block_size, num_kv_heads, head_size)
|
||||
# then convert to HND format (same approach as test_attention_backends.py).
|
||||
kv_cache_raw = torch.zeros(
|
||||
2,
|
||||
# kv_in_head_dim: (B, N, 2*H, hs) — K/V as separate head groups
|
||||
# else: (B, N, H, 2*hs) — K/V packed in content dim
|
||||
n_heads, content = (
|
||||
(2 * num_kv_heads, head_size)
|
||||
if kv_in_head_dim
|
||||
else (num_kv_heads, 2 * head_size)
|
||||
)
|
||||
kv_cache = torch.zeros(
|
||||
num_blocks,
|
||||
block_size,
|
||||
num_kv_heads,
|
||||
head_size,
|
||||
n_heads,
|
||||
content,
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
)
|
||||
kv_cache_flat = kv_cache_raw.view(2, -1, num_kv_heads, head_size)
|
||||
kv_cache_flat = kv_cache.view(-1, n_heads, content)
|
||||
|
||||
start_block_idx = 1
|
||||
for i in range(batch_size):
|
||||
k_ctx, v_ctx = k_contexts[i], v_contexts[i]
|
||||
start = start_block_idx * block_size
|
||||
end = start + k_ctx.shape[0]
|
||||
kv_cache_flat[0, start:end] = k_ctx
|
||||
kv_cache_flat[1, start:end] = v_ctx
|
||||
if kv_in_head_dim:
|
||||
kv_cache_flat[start:end, :num_kv_heads] = k_ctx
|
||||
kv_cache_flat[start:end, num_kv_heads:] = v_ctx
|
||||
else:
|
||||
kv_cache_flat[start:end, :, :head_size] = k_ctx
|
||||
kv_cache_flat[start:end, :, head_size:] = v_ctx
|
||||
start_block_idx += cdiv(int(seq_lens[i]), block_size)
|
||||
|
||||
blocks_end = start_block_idx
|
||||
@@ -142,7 +150,7 @@ def _create_hnd_kv_cache(
|
||||
perm = torch.randperm(blocks_end - 1) + 1
|
||||
inv_perm = torch.zeros(blocks_end, dtype=torch.long, device=device)
|
||||
inv_perm[1:] = torch.argsort(perm) + 1
|
||||
kv_cache_raw[:, 1:blocks_end] = kv_cache_raw[:, perm]
|
||||
kv_cache[1:blocks_end] = kv_cache[perm]
|
||||
|
||||
# Build block table.
|
||||
start_block_idx = 1
|
||||
@@ -165,10 +173,8 @@ def _create_hnd_kv_cache(
|
||||
i, block_indices
|
||||
] * block_size + intra_block_offsets.to(device)
|
||||
|
||||
# Transpose to FlashInfer logical shape then make HND-strided.
|
||||
kv_cache = kv_cache_raw.transpose(0, 1)
|
||||
kv_cache = kv_cache.transpose(2, 3).contiguous().transpose(2, 3)
|
||||
return kv_cache
|
||||
# Transpose to canonical: (B, H, N, 2*hs) or (B, 2*H, N, hs)
|
||||
return kv_cache.transpose(1, 2).contiguous()
|
||||
|
||||
|
||||
def _create_nvfp4_hnd_kv_cache(
|
||||
@@ -187,20 +193,15 @@ def _create_nvfp4_hnd_kv_cache(
|
||||
reshape_and_cache_flash, using the same block-table layout as
|
||||
_create_hnd_kv_cache.
|
||||
|
||||
The returned tensor is dtype ``uint8`` with shape
|
||||
``(num_blocks, 2, block_size, num_kv_heads, full_dim)`` in logical
|
||||
(NHD) order, but physically permuted to HND layout via stride order
|
||||
``(0, 1, 3, 2, 4)`` (i.e. ``num_kv_heads`` before ``block_size``).
|
||||
|
||||
The last dimension ``full_dim = head_size // 2 + head_size // 16``
|
||||
packs two regions contiguously:
|
||||
The returned tensor is dtype ``uint8`` with head-group layout
|
||||
``(num_blocks, 2 * num_kv_heads, block_size, full_dim)``
|
||||
where K heads occupy the first ``num_kv_heads`` heads and V heads the second.
|
||||
Each ``full_dim = head_size // 2 + head_size // 16`` block packs two regions:
|
||||
- **FP4 data** (``head_size // 2`` bytes): pairs of E2M1 values,
|
||||
two per byte.
|
||||
- **FP8 block scales** (``head_size // 16`` bytes): one E4M3
|
||||
scale per 16-element block.
|
||||
|
||||
Dimension 1 indexes K (``[:, 0]``) and V (``[:, 1]``).
|
||||
|
||||
Args:
|
||||
k_contexts: List of key context tensors, one per sequence.
|
||||
v_contexts: List of value context tensors, one per sequence.
|
||||
@@ -219,6 +220,7 @@ def _create_nvfp4_hnd_kv_cache(
|
||||
``torch.Tensor``: The nvfp4 kv_cache tensor (uint8, HND-strided).
|
||||
"""
|
||||
# First create a bf16 HND cache so block tables are populated.
|
||||
# Use kv_in_head_dim=True so K/V are separate head groups (B, 2*H, N, hs).
|
||||
bf16_cache = _create_hnd_kv_cache(
|
||||
k_contexts,
|
||||
v_contexts,
|
||||
@@ -229,20 +231,20 @@ def _create_nvfp4_hnd_kv_cache(
|
||||
device,
|
||||
num_blocks,
|
||||
common_attn_metadata,
|
||||
kv_in_head_dim=True,
|
||||
)
|
||||
|
||||
# Allocate nvfp4 cache: same shape but with full_dim (data + scale).
|
||||
# (num_blocks, 2 * num_kv_heads, block_size, full_dim) — K heads first, then V heads
|
||||
full_dim = nvfp4_kv_cache_full_dim(head_size)
|
||||
hnd_order = (0, 1, 3, 2, 4)
|
||||
nvfp4_cache = torch.zeros(
|
||||
(num_blocks, 2, num_kv_heads, block_size, full_dim),
|
||||
(num_blocks, 2 * num_kv_heads, block_size, full_dim),
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
).permute(*hnd_order)
|
||||
)
|
||||
k_cache, v_cache = nvfp4_cache.split(num_kv_heads, dim=1)
|
||||
|
||||
# Flatten bf16 context into tokens and quantize via reshape_and_cache_flash.
|
||||
# bf16_cache is (num_blocks, 2, block_size, num_kv_heads, head_size) logical
|
||||
# with HND physical strides.
|
||||
# bf16_cache is (B, 2*H, N, hs); split K/V on head dim.
|
||||
block_table = common_attn_metadata.block_table_tensor
|
||||
seq_lens = common_attn_metadata.seq_lens.cpu()
|
||||
query_lens = (
|
||||
@@ -258,19 +260,21 @@ def _create_nvfp4_hnd_kv_cache(
|
||||
# Gather context tokens from the bf16 cache using block table.
|
||||
n_ctx_blocks = (ctx_len + block_size - 1) // block_size
|
||||
blocks = block_table[i, :n_ctx_blocks]
|
||||
# bf16_cache[:, kv_idx] is (num_blocks, block_size, num_kv_heads, head_size)
|
||||
k_ctx = bf16_cache[blocks, 0].reshape(-1, num_kv_heads, head_size)[:ctx_len]
|
||||
v_ctx = bf16_cache[blocks, 1].reshape(-1, num_kv_heads, head_size)[:ctx_len]
|
||||
# bf16_cache is (B, 2*H, N, hs); split K and V head groups.
|
||||
k_bf16, v_bf16 = bf16_cache[blocks].split(num_kv_heads, dim=1)
|
||||
k_ctx = k_bf16.transpose(1, 2).reshape(-1, num_kv_heads, head_size)[:ctx_len]
|
||||
v_ctx = v_bf16.transpose(1, 2).reshape(-1, num_kv_heads, head_size)[:ctx_len]
|
||||
# Build slot mapping for these context tokens.
|
||||
token_offsets = torch.arange(ctx_len, device=device)
|
||||
block_indices = token_offsets // block_size
|
||||
intra_offsets = token_offsets % block_size
|
||||
slots = block_table[i, block_indices] * block_size + intra_offsets
|
||||
# reshape_and_cache_flash expects (B, N, H, D) cache views.
|
||||
torch.ops._C_cache_ops.reshape_and_cache_flash(
|
||||
k_ctx,
|
||||
v_ctx,
|
||||
nvfp4_cache[:, 0],
|
||||
nvfp4_cache[:, 1],
|
||||
k_cache.transpose(1, 2),
|
||||
v_cache.transpose(1, 2),
|
||||
slots,
|
||||
"nvfp4",
|
||||
kv_scale_t,
|
||||
|
||||
@@ -289,10 +289,9 @@ async def test_send_kv_to_decode_aligns_consumer_regions_by_layer_metadata(
|
||||
prefill_worker = prefill_connector.connector_worker
|
||||
|
||||
block_len = 4096
|
||||
kv_half = block_len // 2
|
||||
prefill_worker.kv_caches_base_addr = [0x1000]
|
||||
prefill_worker.block_len_per_layer = [block_len]
|
||||
prefill_worker.kv_block_len_per_layer = [kv_half]
|
||||
prefill_worker.kv_block_len_per_layer = [block_len]
|
||||
prefill_worker.registered_layer_names = ["model.layers.1.self_attn"]
|
||||
prefill_worker.registered_layer_indices = [1]
|
||||
|
||||
@@ -321,7 +320,7 @@ async def test_send_kv_to_decode_aligns_consumer_regions_by_layer_metadata(
|
||||
req_blocks={"d-req-layer-align": (transfer_id, [[20]])},
|
||||
kv_caches_base_addr=[0xA000, 0xB000],
|
||||
block_lens=[block_len, block_len],
|
||||
kv_block_lens=[kv_half, kv_half],
|
||||
kv_block_lens=[block_len, block_len],
|
||||
registered_layer_names=[
|
||||
"model.layers.0.self_attn",
|
||||
"model.layers.1.self_attn",
|
||||
@@ -338,15 +337,9 @@ async def test_send_kv_to_decode_aligns_consumer_regions_by_layer_metadata(
|
||||
await prefill_worker.send_kv_to_decode(identity, mock_socket, xfer_meta)
|
||||
|
||||
src_ptrs, dst_ptrs, lengths = mock_send_blocks.call_args[0][1:]
|
||||
assert src_ptrs == [
|
||||
0x1000 + 10 * block_len,
|
||||
0x1000 + 10 * block_len + kv_half,
|
||||
]
|
||||
assert dst_ptrs == [
|
||||
0xB000 + 20 * block_len,
|
||||
0xB000 + 20 * block_len + kv_half,
|
||||
]
|
||||
assert lengths == [kv_half, kv_half]
|
||||
assert src_ptrs == [0x1000 + 10 * block_len]
|
||||
assert dst_ptrs == [0xB000 + 20 * block_len]
|
||||
assert lengths == [block_len]
|
||||
|
||||
sent_identity, sent_payload = mock_socket.send_multipart.call_args[0][0]
|
||||
assert sent_identity == identity
|
||||
@@ -832,9 +825,8 @@ async def test_kv_producer(monkeypatch):
|
||||
prefill_worker = prefill_connector.connector_worker
|
||||
prefill_worker.kv_caches_base_addr = [0x1000]
|
||||
block_len = 4096
|
||||
kv_half = block_len // 2
|
||||
prefill_worker.block_len_per_layer = [block_len]
|
||||
prefill_worker.kv_block_len_per_layer = [kv_half]
|
||||
prefill_worker.kv_block_len_per_layer = [block_len]
|
||||
prefill_worker.registered_layer_names = ["model.layers.0.self_attn"]
|
||||
prefill_worker.registered_layer_indices = [0]
|
||||
|
||||
@@ -862,7 +854,7 @@ async def test_kv_producer(monkeypatch):
|
||||
req_blocks={"d-req-1": (transfer_id, [[20, 21]])},
|
||||
kv_caches_base_addr=[0x2000],
|
||||
block_lens=[block_len],
|
||||
kv_block_lens=[kv_half],
|
||||
kv_block_lens=[block_len],
|
||||
registered_layer_names=["model.layers.0.self_attn"],
|
||||
registered_layer_indices=[0],
|
||||
)
|
||||
@@ -874,24 +866,18 @@ async def test_kv_producer(monkeypatch):
|
||||
with patch.object(
|
||||
prefill_worker, "_send_blocks", return_value=0
|
||||
) as mock_send_blocks:
|
||||
# With blocks-first layout, each block is virtually split
|
||||
# into K and V halves, producing non-coalesced transfers.
|
||||
def expected_split_transfers(src_base, dst_base, src_blocks, dst_blocks):
|
||||
"""Build expected (src_ptrs, dst_ptrs, lengths) for
|
||||
virtual-split K/V transfers."""
|
||||
src_ptrs, dst_ptrs, lengths = [], [], []
|
||||
for kv_offset in (0, kv_half):
|
||||
for sb, db in zip(src_blocks, dst_blocks):
|
||||
src_ptrs.append(src_base + sb * block_len + kv_offset)
|
||||
dst_ptrs.append(dst_base + db * block_len + kv_offset)
|
||||
lengths.append(kv_half)
|
||||
return src_ptrs, dst_ptrs, lengths
|
||||
|
||||
def expected_transfers(src_base, dst_base, src_blocks, dst_blocks):
|
||||
n = len(src_blocks)
|
||||
return (
|
||||
[src_base + src_blocks[0] * block_len],
|
||||
[dst_base + dst_blocks[0] * block_len],
|
||||
[n * block_len],
|
||||
)
|
||||
|
||||
# Normal case: 2 blocks to 2 blocks
|
||||
await prefill_worker.send_kv_to_decode(identity, mock_socket, xfer_meta)
|
||||
src, dst, lens = expected_split_transfers(
|
||||
0x1000, 0x2000, [10, 11], [20, 21]
|
||||
)
|
||||
src, dst, lens = expected_transfers(0x1000, 0x2000, [10, 11], [20, 21])
|
||||
mock_send_blocks.assert_called_once_with(
|
||||
"consumer-host:54321",
|
||||
src,
|
||||
@@ -923,7 +909,7 @@ async def test_kv_producer(monkeypatch):
|
||||
# Worker processes the consumer's request
|
||||
await prefill_worker.send_kv_to_decode(identity, mock_socket, xfer_meta)
|
||||
# Verify transfer parameters are correct: 11 to 20
|
||||
src, dst, lens = expected_split_transfers(0x1000, 0x2000, [11], [20])
|
||||
src, dst, lens = expected_transfers(0x1000, 0x2000, [11], [20])
|
||||
mock_send_blocks.assert_called_once_with(
|
||||
"consumer-host:54321",
|
||||
src,
|
||||
@@ -1266,7 +1252,7 @@ async def test_kv_producer_heterogeneous_tp(monkeypatch, d_tp_size):
|
||||
|
||||
prefill_worker.kv_caches_base_addr = [0x1000]
|
||||
prefill_worker.block_len_per_layer = [local_block_len]
|
||||
prefill_worker.kv_block_len_per_layer = [local_block_len // 2]
|
||||
prefill_worker.kv_block_len_per_layer = [local_block_len]
|
||||
prefill_worker.registered_layer_names = ["model.layers.0.self_attn"]
|
||||
prefill_worker.registered_layer_indices = [0]
|
||||
|
||||
@@ -1314,7 +1300,7 @@ async def test_kv_producer_heterogeneous_tp(monkeypatch, d_tp_size):
|
||||
},
|
||||
kv_caches_base_addr=[0x2000],
|
||||
block_lens=[remote_block_len],
|
||||
kv_block_lens=[remote_block_len // 2],
|
||||
kv_block_lens=[remote_block_len],
|
||||
registered_layer_names=["model.layers.0.self_attn"],
|
||||
registered_layer_indices=[0],
|
||||
)
|
||||
@@ -1336,47 +1322,31 @@ async def test_kv_producer_heterogeneous_tp(monkeypatch, d_tp_size):
|
||||
flat_remote = [b for g in remote_block_ids for b in g]
|
||||
num_blocks = len(flat_local)
|
||||
|
||||
# With blocks-first layout, virtual split halves block
|
||||
# lengths and doubles transfer regions (K + V).
|
||||
local_kv_block_len = local_block_len // 2
|
||||
remote_kv_block_len = remote_block_len // 2
|
||||
assert len(src_ptrs) == num_blocks
|
||||
assert len(dst_ptrs) == num_blocks
|
||||
assert len(lengths) == num_blocks
|
||||
|
||||
assert len(src_ptrs) == 2 * num_blocks
|
||||
assert len(dst_ptrs) == 2 * num_blocks
|
||||
assert len(lengths) == 2 * num_blocks
|
||||
|
||||
# Compute expected offsets using kv_block_len
|
||||
if d_tp_size <= P_TP_SIZE:
|
||||
tp_ratio = P_TP_SIZE // d_tp_size
|
||||
expected_src_off = 0
|
||||
expected_dst_off = (P_TP_RANK % tp_ratio) * local_kv_block_len
|
||||
expected_xfer_len = local_kv_block_len
|
||||
expected_dst_off = (P_TP_RANK % tp_ratio) * local_block_len
|
||||
expected_xfer_len = local_block_len
|
||||
else:
|
||||
ratio_abs = d_tp_size // P_TP_SIZE
|
||||
expected_src_off = (d_rank % ratio_abs) * remote_kv_block_len
|
||||
expected_src_off = (d_rank % ratio_abs) * remote_block_len
|
||||
expected_dst_off = 0
|
||||
expected_xfer_len = remote_kv_block_len
|
||||
expected_xfer_len = remote_block_len
|
||||
|
||||
# First num_blocks entries are K region,
|
||||
# next num_blocks are V region.
|
||||
for region_idx in range(2):
|
||||
local_region_base = 0x1000 + region_idx * local_kv_block_len
|
||||
remote_region_base = 0x2000 + region_idx * remote_kv_block_len
|
||||
for blk_idx, (lblk, rblk) in enumerate(
|
||||
zip(flat_local, flat_remote)
|
||||
):
|
||||
idx = region_idx * num_blocks + blk_idx
|
||||
assert src_ptrs[idx] == (
|
||||
local_region_base
|
||||
+ lblk * local_block_len
|
||||
+ expected_src_off
|
||||
)
|
||||
assert dst_ptrs[idx] == (
|
||||
remote_region_base
|
||||
+ rblk * remote_block_len
|
||||
+ expected_dst_off
|
||||
)
|
||||
assert lengths[idx] == expected_xfer_len
|
||||
local_region_base = 0x1000
|
||||
remote_region_base = 0x2000
|
||||
for blk_idx, (lblk, rblk) in enumerate(zip(flat_local, flat_remote)):
|
||||
assert src_ptrs[blk_idx] == (
|
||||
local_region_base + lblk * local_block_len + expected_src_off
|
||||
)
|
||||
assert dst_ptrs[blk_idx] == (
|
||||
remote_region_base + rblk * remote_block_len + expected_dst_off
|
||||
)
|
||||
assert lengths[blk_idx] == expected_xfer_len
|
||||
|
||||
# Verify successful response sent back to consumer
|
||||
mock_socket.send_multipart.assert_called_once()
|
||||
|
||||
@@ -1068,6 +1068,65 @@ class TestNixlHandshake:
|
||||
# whole block is moved.
|
||||
worker.add_remote_agent(meta, remote_tp_size=1)
|
||||
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
|
||||
FakeNixlWrapper,
|
||||
)
|
||||
def test_hybrid_mamba_attention_remote_descs_use_packed_head_slices(
|
||||
self, default_vllm_config, dist_init
|
||||
):
|
||||
worker = FakeNixlConnectorWorker(
|
||||
create_vllm_config(), "engine", hand_shake_latency=0
|
||||
)
|
||||
|
||||
remote_block_len = 2048
|
||||
local_block_len = remote_block_len // 2
|
||||
worker.block_len_per_layer = [local_block_len]
|
||||
worker._region_is_mla = [False]
|
||||
worker.num_blocks = 1
|
||||
worker.num_regions = 1
|
||||
worker._has_mamba = True
|
||||
worker._mamba_ssm_size = (128, 256)
|
||||
worker.transfer_topo = TransferTopology(
|
||||
tp_rank=1,
|
||||
tp_size=2,
|
||||
block_size=worker.block_size,
|
||||
engine_id=worker.engine_id,
|
||||
is_mla=False,
|
||||
is_mamba=True,
|
||||
total_num_kv_heads=2,
|
||||
attn_backends=worker.attn_backends,
|
||||
tensor_shape=None,
|
||||
)
|
||||
assert worker.transfer_topo.virtually_split_kv_in_blocks
|
||||
|
||||
plan = MagicMock(
|
||||
source_ranks_per_group=((0,), (0,)),
|
||||
rank_offset_factor=1,
|
||||
)
|
||||
meta = MagicMock(
|
||||
kv_caches_base_addr=[0x1000],
|
||||
device_id=0,
|
||||
num_blocks=1,
|
||||
block_lens=[remote_block_len],
|
||||
)
|
||||
|
||||
assert worker.get_backend_aware_kv_block_len(0, mamba_view=False) == (
|
||||
local_block_len
|
||||
)
|
||||
assert (
|
||||
worker.get_backend_aware_kv_block_len(0, first_split=True, mamba_view=True)
|
||||
== worker._mamba_ssm_size[0]
|
||||
)
|
||||
assert (
|
||||
worker.get_backend_aware_kv_block_len(0, first_split=False, mamba_view=True)
|
||||
== worker._mamba_ssm_size[1]
|
||||
)
|
||||
|
||||
assert worker._build_fa_remote(plan, meta, block_size_ratio=1) == [
|
||||
(0x1000 + local_block_len, local_block_len, 0)
|
||||
]
|
||||
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
|
||||
FakeNixlWrapper,
|
||||
@@ -1691,13 +1750,6 @@ def _run_abort_timeout_test(llm: LLM, timeout: int):
|
||||
reason="Attention backend FLASH_ATTN is not supported on ROCm",
|
||||
),
|
||||
),
|
||||
pytest.param(
|
||||
"ROCM_ATTN",
|
||||
marks=pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="Attention backend ROCM_ATTN is only supported on ROCm",
|
||||
),
|
||||
),
|
||||
"TRITON_ATTN",
|
||||
],
|
||||
)
|
||||
@@ -1824,8 +1876,7 @@ def test_register_kv_caches(
|
||||
test_shape = backend_cls.get_kv_cache_shape(
|
||||
num_blocks=1, block_size=16, num_kv_heads=1, head_size=1
|
||||
)
|
||||
is_blocks_first = len(test_shape) == 5 and test_shape[0] == 1
|
||||
virtually_split = is_blocks_first and not connector.prefer_cross_layer_blocks
|
||||
is_blocks_first = len(test_shape) == 4 and test_shape[0] == 1
|
||||
|
||||
if connector.prefer_cross_layer_blocks:
|
||||
with set_current_vllm_config(vllm_config):
|
||||
@@ -1856,7 +1907,7 @@ def test_register_kv_caches(
|
||||
]
|
||||
expected_num_entries = 1
|
||||
|
||||
expected_blocks_count = num_blocks * (2 if virtually_split else 1)
|
||||
expected_blocks_count = num_blocks
|
||||
|
||||
kv_caches = {"all-layers": cross_layers_kv_cache}
|
||||
else:
|
||||
@@ -1885,6 +1936,7 @@ def test_register_kv_caches(
|
||||
unique_tensor.data_ptr(),
|
||||
]
|
||||
expected_num_entries = 2
|
||||
expected_blocks_count = kv_cache_config.num_blocks * 2
|
||||
else:
|
||||
expected_tensor_size = (
|
||||
shared_tensor[0].element_size() * shared_tensor[0].numel()
|
||||
@@ -1896,7 +1948,7 @@ def test_register_kv_caches(
|
||||
unique_tensor[1].data_ptr(),
|
||||
]
|
||||
expected_num_entries = 4
|
||||
expected_blocks_count = kv_cache_config.num_blocks * 4
|
||||
expected_blocks_count = kv_cache_config.num_blocks * 4
|
||||
|
||||
# Execute register_kv_caches
|
||||
connector.register_kv_caches(kv_caches)
|
||||
@@ -1930,10 +1982,7 @@ def test_register_kv_caches(
|
||||
else:
|
||||
num_blocks = kv_cache_config.num_blocks
|
||||
|
||||
if virtually_split:
|
||||
expected_block_len = expected_tensor_size // num_blocks // 2
|
||||
else:
|
||||
expected_block_len = expected_tensor_size // num_blocks
|
||||
expected_block_len = expected_tensor_size // num_blocks
|
||||
|
||||
for i, block_entry in enumerate(blocks_data):
|
||||
block_start_addr, block_len, tp_rank = block_entry
|
||||
|
||||
@@ -18,8 +18,8 @@ class _FakeAttentionBackend:
|
||||
block_size: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
) -> tuple[int, int, int, int, int]:
|
||||
return (2, num_blocks, num_kv_heads, block_size, head_size)
|
||||
) -> tuple[int, int, int, int]:
|
||||
return (num_blocks, num_kv_heads, block_size, 2 * head_size)
|
||||
|
||||
|
||||
def _make_topology(
|
||||
|
||||
@@ -795,9 +795,10 @@ def test_kv_cache_stride_order(monkeypatch, model_runner):
|
||||
)
|
||||
|
||||
# TODO mla test
|
||||
default_stride = tuple(range(5))
|
||||
default_stride = tuple(range(len(expected_kv_cache_shape)))
|
||||
non_default_stride = (*default_stride[1:], default_stride[0])
|
||||
# Permutation that gets you back to expected kv shape
|
||||
for test_stride in ((1, 4, 0, 2, 3), (0, 1, 2, 3, 4)):
|
||||
for test_stride in (non_default_stride, default_stride):
|
||||
|
||||
def rnd_stride_order(
|
||||
include_num_layers_dimension: bool = False, test_stride=test_stride
|
||||
@@ -1300,9 +1301,10 @@ def test_hybrid_attention_mamba_tensor_shapes():
|
||||
actual_kv = vllm_ctx[layer].kv_cache[kernel_block, :]
|
||||
expected = attn_blocks_constant[i]
|
||||
|
||||
# Check K and V separately
|
||||
assert torch.equal(actual_kv[0], expected)
|
||||
assert torch.equal(actual_kv[1], expected)
|
||||
# Packed layout: (num_kv_heads, block_size, 2*head_size). Every
|
||||
# head in the block was filled with the same constant.
|
||||
for head_idx in range(actual_kv.shape[0]):
|
||||
assert torch.equal(actual_kv[head_idx], expected)
|
||||
|
||||
for layer in [layer_2, layer_3, layer_4, layer_5]:
|
||||
for i, kv_block in enumerate(kv_blocks_for_mamba):
|
||||
|
||||
@@ -10,7 +10,12 @@ from typing import TYPE_CHECKING, Any, Literal, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig, get_current_vllm_config, get_layers_from_vllm_config
|
||||
from vllm.config import (
|
||||
VllmConfig,
|
||||
get_current_vllm_config,
|
||||
get_layers_from_vllm_config,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
@@ -344,14 +349,15 @@ def get_current_attn_backends(
|
||||
)
|
||||
from vllm.v1.attention.selector import get_attn_backend
|
||||
|
||||
return [
|
||||
get_attn_backend(
|
||||
head_size=vllm_config.model_config.get_head_size(),
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
kv_cache_dtype=vllm_config.cache_config.cache_dtype,
|
||||
use_mla=vllm_config.model_config.use_mla,
|
||||
)
|
||||
]
|
||||
with set_current_vllm_config(vllm_config):
|
||||
return [
|
||||
get_attn_backend(
|
||||
head_size=vllm_config.model_config.get_head_size(),
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
kv_cache_dtype=vllm_config.cache_config.cache_dtype,
|
||||
use_mla=vllm_config.model_config.use_mla,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def get_current_attn_backend(
|
||||
@@ -426,12 +432,16 @@ class TransferTopology:
|
||||
head_size=1,
|
||||
)
|
||||
logger.debug("Test kv_cache_shape: %s", kv_cache_shape)
|
||||
# Non-MLA backends caches have 5 dims [num_blocks, 2, H,N,D],
|
||||
# we just mock num_blocks to 1 for the dimension check below.
|
||||
# Hybrid SSM models assume a single blocks_first layout
|
||||
self._is_kv_layout_blocks_first = self.is_mamba or (
|
||||
len(kv_cache_shape) == 5 and kv_cache_shape[0] == 1
|
||||
)
|
||||
assert kv_cache_shape[0] == 1, (
|
||||
"KV cache layout must be blocks-first; expected mocked "
|
||||
f"num_blocks=1 in leading dim, got shape {kv_cache_shape}."
|
||||
)
|
||||
if not self.is_mla:
|
||||
assert len(kv_cache_shape) == 4, (
|
||||
"Attention KV cache layout must be standardized as "
|
||||
"[num_blocks, num_kv_heads, block_size, content_size], "
|
||||
f"got shape {kv_cache_shape}."
|
||||
)
|
||||
|
||||
self._cross_layers_blocks = False
|
||||
if self.tensor_shape is not None:
|
||||
@@ -490,29 +500,19 @@ class TransferTopology:
|
||||
# Layout properties
|
||||
# ============================================================
|
||||
|
||||
@property
|
||||
def is_kv_layout_blocks_first(self) -> bool:
|
||||
return self._is_kv_layout_blocks_first
|
||||
|
||||
@property
|
||||
def cross_layers_blocks(self) -> bool:
|
||||
return self._cross_layers_blocks
|
||||
|
||||
@property
|
||||
def virtually_split_kv_in_blocks(self) -> bool:
|
||||
# Whether to logically split each block into K and V halves.
|
||||
# Applies when K/V are interleaved within each block (blocks-first),
|
||||
# but NOT when cross-layer blocks are used — cross-layer blocks have
|
||||
# per-layer K/V interleaving (L0_K, L0_V, L1_K, L1_V, ...) so a
|
||||
# simple half-split does not separate K from V.
|
||||
return self._is_kv_layout_blocks_first and not self._cross_layers_blocks
|
||||
|
||||
@property
|
||||
def split_k_and_v(self) -> bool:
|
||||
# Whether to register regions for K and V separately (when present).
|
||||
return not (
|
||||
self._cross_layers_blocks or self.is_mla or self.is_kv_layout_blocks_first
|
||||
)
|
||||
# Whether to logically split each block into two separately-indexable
|
||||
# sub-regions. With K and V packed into the content dim, an attention
|
||||
# block transfers as a single unit — no K/V sub-split is needed. Only
|
||||
# Mamba still needs this, to index its two state regions (conv/ssm)
|
||||
# separately. Not applicable to cross-layer blocks (per-layer
|
||||
# interleaving means a simple half-split does not separate the parts).
|
||||
return self.is_mamba and not self._cross_layers_blocks
|
||||
|
||||
# ============================================================
|
||||
# Common methods
|
||||
@@ -616,8 +616,9 @@ class TransferTopology:
|
||||
# Swap [2<>num_blocks] dims for hybrid SSM layout.
|
||||
cache = cache.transpose(0, 1)
|
||||
|
||||
# Regular case: backends like FA register K/V in separate regions
|
||||
return cache if self.split_k_and_v else [cache]
|
||||
# K and V are packed into one tensor (content dim), so each layer
|
||||
# registers as a single region.
|
||||
return [cache]
|
||||
|
||||
def describe(self, remote_engine_id: EngineId, remote_pp_rank: int = 0) -> str:
|
||||
"""One-line summary of transfer config for logging."""
|
||||
|
||||
@@ -7,6 +7,7 @@ from typing import NamedTuple
|
||||
import torch
|
||||
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheSpec,
|
||||
MLAAttentionSpec,
|
||||
@@ -56,6 +57,11 @@ def is_mla_cache_layer(
|
||||
return isinstance(spec, (MLAAttentionSpec, SlidingWindowMLASpec))
|
||||
|
||||
|
||||
def _content_packed_dim(spec: AttentionSpec) -> int:
|
||||
head_size_v = getattr(spec, "head_size_v", spec.head_size)
|
||||
return spec.head_size + head_size_v
|
||||
|
||||
|
||||
def _spec_dim_matches(value: int, expected: int | None) -> bool:
|
||||
return expected is None or value == expected
|
||||
|
||||
@@ -223,6 +229,30 @@ def get_layer_transfer_geometry(
|
||||
split_kv_regions=False,
|
||||
)
|
||||
|
||||
if (
|
||||
not is_mla_cache
|
||||
and isinstance(spec, AttentionSpec)
|
||||
and len(shape) == 4
|
||||
and shape[1] == spec.num_kv_heads
|
||||
and shape[2] == spec.block_size
|
||||
and shape[3] == _content_packed_dim(spec)
|
||||
):
|
||||
num_blocks, num_kv_heads, block_size, packed_dim = shape
|
||||
slot_size_bytes = num_kv_heads * packed_dim * element_size
|
||||
block_len = block_size * slot_size_bytes
|
||||
return LayerTransferGeometry(
|
||||
num_blocks=num_blocks,
|
||||
block_size=block_size,
|
||||
block_len=block_len,
|
||||
slot_size_bytes=slot_size_bytes,
|
||||
block_stride=stride[0],
|
||||
local_kv_stride=None,
|
||||
remote_kv_stride=None,
|
||||
transfers_per_block=1,
|
||||
regions_per_block=1,
|
||||
split_kv_regions=False,
|
||||
)
|
||||
|
||||
cache_kind = "MLA" if is_mla_cache else "K/V"
|
||||
raise ValueError(
|
||||
f"Unsupported MoRIIO {cache_kind} cache shape for layer "
|
||||
|
||||
@@ -216,12 +216,10 @@ class NixlBaseConnectorWorker:
|
||||
if n_regions == 0 or self.num_regions == 0:
|
||||
return [False] * num_fa_descs
|
||||
nblk = num_fa_descs // self.num_regions
|
||||
virtually_split = self.transfer_topo.virtually_split_kv_in_blocks
|
||||
flags: list[bool] = []
|
||||
for i in range(n_regions):
|
||||
replicated = self._is_region_replicated(i)
|
||||
num_streams = 1 if replicated or not virtually_split else 2
|
||||
flags.extend([replicated] * (num_streams * nblk))
|
||||
flags.extend([replicated] * nblk)
|
||||
assert len(flags) == num_fa_descs, (
|
||||
f"FA desc flags {len(flags)} != num_fa_descs {num_fa_descs}"
|
||||
)
|
||||
@@ -712,31 +710,31 @@ class NixlBaseConnectorWorker:
|
||||
NOT directly supported by NIXL (e.g., tpu)
|
||||
"""
|
||||
xfer_buffers: dict[str, torch.Tensor] = {}
|
||||
inv_order = [0, 1, 3, 2, 4]
|
||||
try:
|
||||
for layer_name, kv_cache in kv_caches.items():
|
||||
kv_shape = kv_cache.shape
|
||||
kv_dtype = kv_cache.dtype
|
||||
permute_shape = False
|
||||
if (
|
||||
self.kv_cache_layout == "NHD"
|
||||
and self.vllm_config.kv_transfer_config is not None
|
||||
and self.vllm_config.kv_transfer_config.enable_permute_local_kv
|
||||
):
|
||||
logger.info_once(
|
||||
"'enable_permute_local_kv' flag is enabled while "
|
||||
"device KV Layout is NHD. Init host buffer with"
|
||||
" HND to better support Decode/Prefill TP_ratio > 1."
|
||||
)
|
||||
# Since NHD will not support Decode/Prefill TP_ratio > 1,
|
||||
# we can leverage host_buffer for permute
|
||||
self.host_buffer_kv_cache_layout = "HND"
|
||||
kv_shape = (
|
||||
tuple(kv_shape[i] for i in inv_order)
|
||||
if not self.use_mla
|
||||
else kv_shape
|
||||
)
|
||||
permute_shape = not self.use_mla
|
||||
inv_order = (0, 2, 1, 3)
|
||||
if not self.use_mla:
|
||||
assert kv_cache.ndim == 4
|
||||
|
||||
if self.kv_cache_layout == "NHD":
|
||||
if self.kv_transfer_config.enable_permute_local_kv:
|
||||
logger.info_once(
|
||||
"'enable_permute_local_kv' flag is enabled while "
|
||||
"device KV Layout is NHD. Init host buffer with"
|
||||
" HND to better support Decode/Prefill TP_ratio > 1."
|
||||
)
|
||||
# Since NHD will not support Decode/Prefill TP_ratio > 1,
|
||||
# we can leverage host_buffer for permute.
|
||||
self.host_buffer_kv_cache_layout = "HND"
|
||||
else:
|
||||
# Packed KV layout is logical (B, H, N, 2*D). Allocate
|
||||
# (B, N, H, 2*D) and view it as logical (B, H, N, 2*D)
|
||||
# so raw NIXL transfers see NHD physical strides.
|
||||
kv_shape = tuple(kv_shape[i] for i in inv_order)
|
||||
permute_shape = True
|
||||
|
||||
xfer_buffers[layer_name] = torch.empty(
|
||||
kv_shape, dtype=kv_dtype, device="cpu"
|
||||
@@ -1151,9 +1149,7 @@ class NixlBaseConnectorWorker:
|
||||
f"backend={self.backend_name}, "
|
||||
"all_backends="
|
||||
f"{[backend.get_name() for backend in self.attn_backends]}, "
|
||||
f"kv_cache_layout={self.kv_cache_layout}, "
|
||||
"blocks_first="
|
||||
f"{self.transfer_topo.is_kv_layout_blocks_first}"
|
||||
f"kv_cache_layout={self.kv_cache_layout}"
|
||||
)
|
||||
|
||||
# Need to make sure the device ID is non-negative for NIXL,
|
||||
@@ -1184,22 +1180,6 @@ class NixlBaseConnectorWorker:
|
||||
regions_per_layer = self.num_regions // num_local_layers
|
||||
self._remote_region_offset = regions_per_layer * start_layer
|
||||
|
||||
if self.transfer_topo.virtually_split_kv_in_blocks:
|
||||
# NOTE (NickLucche) When FlashInfer is used, memory is registered
|
||||
# with joint KV for each block. This minimizes the overhead in
|
||||
# registerMem allowing faster descs queries. In order to be able to
|
||||
# split on kv_heads dim as required by heterogeneous TP, one must
|
||||
# be able to index K/V separately. Hence we double the number
|
||||
# of 'virtual' regions here and halve `block_len` below.
|
||||
# Similarly for Mamba layers, we register SSM+Conv as a single region and
|
||||
# then duplicate it logically to be able to index SSM/Conv separately.
|
||||
# Exception: key-only REPLICATE regions (MLA) have no V half, so
|
||||
# they contribute a single desc stream and are not doubled.
|
||||
self.num_regions = sum(
|
||||
1 if self._is_region_replicated(i) else 2
|
||||
for i in range(len(self._region_is_mla))
|
||||
)
|
||||
|
||||
# Total local FA descriptors (boundary between FA and mamba descs).
|
||||
self.num_descs = self.num_regions * self.num_blocks
|
||||
|
||||
@@ -1363,22 +1343,6 @@ class NixlBaseConnectorWorker:
|
||||
block_offset = block_id * page_stride
|
||||
addr = base_addr + block_offset
|
||||
result.append((addr, kv_block_len, self.device_id))
|
||||
|
||||
if (
|
||||
self.transfer_topo.virtually_split_kv_in_blocks
|
||||
and not self._is_region_replicated(i)
|
||||
):
|
||||
# Separate and interleave K/V regions to maintain the same
|
||||
# descs ordering. This is needed for selecting contiguous heads
|
||||
# when split across TP ranks. (Skipped for key-only REPLICATE.)
|
||||
second_split = self.get_backend_aware_kv_block_len(
|
||||
layer_idx=i, first_split=False, mamba_view=False
|
||||
)
|
||||
for block_id in range(num_blocks):
|
||||
block_offset = block_id * page_stride
|
||||
addr = base_addr + block_offset
|
||||
v_addr = addr + kv_block_len
|
||||
result.append((v_addr, second_split, self.device_id))
|
||||
return result
|
||||
|
||||
def _build_fa_remote(
|
||||
@@ -1423,20 +1387,6 @@ class NixlBaseConnectorWorker:
|
||||
# tp rank of size local_block_len.
|
||||
addr = base_addr + block_offset + rank_offset
|
||||
result.append((addr, local_block_len, nixl_agent_meta.device_id))
|
||||
|
||||
emits_v = self.transfer_topo.virtually_split_kv_in_blocks and not replicated
|
||||
if emits_v:
|
||||
# With FlashInfer index V separately to allow head splitting.
|
||||
second_split = self.get_backend_aware_kv_block_len(
|
||||
layer_idx=i, first_split=False, mamba_view=False
|
||||
)
|
||||
second_split = second_split // num_reads
|
||||
for block_id in range(num_blocks):
|
||||
block_offset = block_id * page_size
|
||||
addr = base_addr + block_offset + rank_offset
|
||||
# Hop over the first split of remote page, K, to read V.
|
||||
v_addr = addr + nixl_agent_meta.block_lens[i] // 2
|
||||
result.append((v_addr, second_split, nixl_agent_meta.device_id))
|
||||
return result
|
||||
|
||||
def register_local_xfer_handler(
|
||||
@@ -1927,24 +1877,18 @@ class NixlBaseConnectorWorker:
|
||||
block_size_ratio,
|
||||
)
|
||||
|
||||
split_k_and_v = self.transfer_topo.split_k_and_v
|
||||
|
||||
for block_ids in block_ids_list:
|
||||
indices = torch.tensor(block_ids, device=self.device_type, dtype=torch.long)
|
||||
|
||||
for _, cache_or_caches in self.device_kv_caches.items():
|
||||
cache_list = cache_or_caches if split_k_and_v else [cache_or_caches]
|
||||
for cache in cache_list:
|
||||
if self.enable_permute_local_kv and block_size_ratio > 1:
|
||||
kv_postprocess_blksize_and_layout_on_receive(
|
||||
cache, indices, block_size_ratio
|
||||
)
|
||||
elif self.enable_permute_local_kv:
|
||||
kv_postprocess_layout_on_receive(cache, indices)
|
||||
else:
|
||||
kv_postprocess_blksize_on_receive(
|
||||
cache, indices, block_size_ratio
|
||||
)
|
||||
for cache in self.device_kv_caches.values():
|
||||
if self.enable_permute_local_kv and block_size_ratio > 1:
|
||||
kv_postprocess_blksize_and_layout_on_receive(
|
||||
cache, indices, block_size_ratio
|
||||
)
|
||||
elif self.enable_permute_local_kv:
|
||||
kv_postprocess_layout_on_receive(cache, indices)
|
||||
else:
|
||||
kv_postprocess_blksize_on_receive(cache, indices, block_size_ratio)
|
||||
|
||||
def post_process_device_kv_on_receive_heterogeneous_attn(
|
||||
self, block_ids: list[int]
|
||||
@@ -2398,12 +2342,10 @@ class NixlBaseConnectorWorker:
|
||||
|1st_split-2nd_split| |1st_split-2nd_split |
|
||||
"""
|
||||
assert self.transfer_topo is not None
|
||||
virtually_split = self.transfer_topo.virtually_split_kv_in_blocks
|
||||
if virtually_split and mamba_view:
|
||||
if self.transfer_topo.virtually_split_kv_in_blocks and mamba_view:
|
||||
block_len = self._mamba_ssm_size[not first_split]
|
||||
else:
|
||||
half_block = virtually_split and not self._is_region_replicated(layer_idx)
|
||||
block_len = self.block_len_per_layer[layer_idx] // (2 if half_block else 1)
|
||||
block_len = self.block_len_per_layer[layer_idx]
|
||||
return block_len
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
|
||||
@@ -8,7 +8,8 @@ equal the sparse block size (128), so one selected block maps to exactly one
|
||||
page.
|
||||
|
||||
Main K/V cache layout (vLLM):
|
||||
``(num_blocks, 2, 128, num_kv_heads, head_dim)`` K=[:,0] V=[:,1]
|
||||
``(num_blocks, num_kv_heads, 128, 2 * head_dim)``
|
||||
K=[..., :head_dim] V=[..., head_dim:]
|
||||
|
||||
Only the paths MiniMax M3 uses are implemented: no attention sink, base-2
|
||||
(exp2/log2) softmax. The decode kernels use split-K (flash-decoding) over the
|
||||
@@ -50,7 +51,7 @@ _FP8_DTYPES = (
|
||||
@triton.jit(do_not_specialize_on_alignment=["seq_lens", "prefix_lens"])
|
||||
def _gqa_sparse_fwd_kernel(
|
||||
q_ptr, # [total_q, num_heads, head_dim]
|
||||
kv_cache_ptr, # main cache: [num_blocks, 2, 128, num_kv_heads, head_dim]
|
||||
kv_cache_ptr, # main cache: [num_blocks, num_kv_heads, 128, 2*head_dim]
|
||||
t_ptr, # topk_idx: [num_kv_heads, total_q, topk]
|
||||
o_ptr, # [total_q, num_heads, head_dim]
|
||||
block_table_ptr, # [num_reqs, max_blocks]
|
||||
@@ -68,9 +69,8 @@ def _gqa_sparse_fwd_kernel(
|
||||
stride_qh,
|
||||
stride_qd,
|
||||
stride_kv_blk,
|
||||
stride_kv_kv,
|
||||
stride_kv_pos,
|
||||
stride_kv_h,
|
||||
stride_kv_pos,
|
||||
stride_kv_d,
|
||||
stride_th,
|
||||
stride_tn,
|
||||
@@ -140,9 +140,8 @@ def _gqa_sparse_fwd_kernel(
|
||||
k = tl.load(
|
||||
kv_cache_ptr
|
||||
+ page * stride_kv_blk
|
||||
+ 0 * stride_kv_kv
|
||||
+ off_n[None, :] * stride_kv_pos
|
||||
+ pid_kh * stride_kv_h
|
||||
+ off_n[None, :] * stride_kv_pos
|
||||
+ off_d[:, None] * stride_kv_d,
|
||||
mask=d_mask[:, None] & pos_mask[None, :],
|
||||
other=0.0,
|
||||
@@ -162,10 +161,9 @@ def _gqa_sparse_fwd_kernel(
|
||||
v = tl.load(
|
||||
kv_cache_ptr
|
||||
+ page * stride_kv_blk
|
||||
+ 1 * stride_kv_kv
|
||||
+ off_n[:, None] * stride_kv_pos
|
||||
+ pid_kh * stride_kv_h
|
||||
+ off_d[None, :] * stride_kv_d,
|
||||
+ off_n[:, None] * stride_kv_pos
|
||||
+ (head_dim + off_d[None, :]) * stride_kv_d,
|
||||
mask=pos_mask[:, None] & d_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
@@ -206,7 +204,7 @@ def _gqa_sparse_fwd_kernel(
|
||||
@triton.jit(do_not_specialize=["decode_query_len"])
|
||||
def _gqa_sparse_decode_kernel(
|
||||
q_ptr, # [total_q, num_heads, head_dim]
|
||||
kv_cache_ptr, # main cache: [num_blocks, 2, 128, num_kv_heads, head_dim]
|
||||
kv_cache_ptr, # main cache: [num_blocks, num_kv_heads, 128, 2*head_dim]
|
||||
t_ptr, # topk_idx: [num_kv_heads, total_q, topk]
|
||||
o_ptr, # partial out: [NUM_TOPK_CHUNKS, total_q, num_heads, head_dim]
|
||||
lse_ptr, # partial lse (log2): [NUM_TOPK_CHUNKS, total_q, num_heads]
|
||||
@@ -222,9 +220,8 @@ def _gqa_sparse_decode_kernel(
|
||||
stride_qh,
|
||||
stride_qd,
|
||||
stride_kv_blk,
|
||||
stride_kv_kv,
|
||||
stride_kv_pos,
|
||||
stride_kv_h,
|
||||
stride_kv_pos,
|
||||
stride_kv_d,
|
||||
stride_th,
|
||||
stride_tn,
|
||||
@@ -300,9 +297,8 @@ def _gqa_sparse_decode_kernel(
|
||||
k = tl.load(
|
||||
kv_cache_ptr
|
||||
+ page * stride_kv_blk
|
||||
+ 0 * stride_kv_kv
|
||||
+ off_n[None, :] * stride_kv_pos
|
||||
+ pid_kh * stride_kv_h
|
||||
+ off_n[None, :] * stride_kv_pos
|
||||
+ off_d[:, None] * stride_kv_d,
|
||||
mask=d_mask[:, None] & pos_mask[None, :],
|
||||
other=0.0,
|
||||
@@ -319,10 +315,9 @@ def _gqa_sparse_decode_kernel(
|
||||
v = tl.load(
|
||||
kv_cache_ptr
|
||||
+ page * stride_kv_blk
|
||||
+ 1 * stride_kv_kv
|
||||
+ off_n[:, None] * stride_kv_pos
|
||||
+ pid_kh * stride_kv_h
|
||||
+ off_d[None, :] * stride_kv_d,
|
||||
+ off_n[:, None] * stride_kv_pos
|
||||
+ (head_dim + off_d[None, :]) * stride_kv_d,
|
||||
mask=pos_mask[:, None] & d_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
@@ -418,7 +413,7 @@ def _merge_topk_attn_out_kernel(
|
||||
@torch.no_grad()
|
||||
def minimax_m3_sparse_attn(
|
||||
q: torch.Tensor, # [total_q, num_heads, head_dim]
|
||||
kv_cache: torch.Tensor, # [num_blocks, 2, 128, num_kv_heads, head_dim]
|
||||
kv_cache: torch.Tensor, # [num_blocks, num_kv_heads, 128, 2*head_dim]
|
||||
topk_idx: torch.Tensor, # [num_kv_heads, total_q, topk]
|
||||
block_table: torch.Tensor, # [batch, max_blocks]
|
||||
cu_seqlens_q: torch.Tensor, # [batch+1] int32
|
||||
@@ -459,7 +454,6 @@ def minimax_m3_sparse_attn(
|
||||
kv_cache.stride(1),
|
||||
kv_cache.stride(2),
|
||||
kv_cache.stride(3),
|
||||
kv_cache.stride(4),
|
||||
topk_idx.stride(0),
|
||||
topk_idx.stride(1),
|
||||
topk_idx.stride(2),
|
||||
@@ -476,7 +470,7 @@ def minimax_m3_sparse_attn(
|
||||
@torch.no_grad()
|
||||
def minimax_m3_sparse_attn_decode(
|
||||
q: torch.Tensor, # [total_q, num_heads, head_dim]
|
||||
kv_cache: torch.Tensor, # [num_blocks, 2, 128, num_kv_heads, head_dim]
|
||||
kv_cache: torch.Tensor, # [num_blocks, num_kv_heads, 128, 2*head_dim]
|
||||
topk_idx: torch.Tensor, # [num_kv_heads, total_q, topk]
|
||||
block_table: torch.Tensor, # [num_reqs, max_blocks]
|
||||
seq_lens: torch.Tensor, # [num_reqs] int32
|
||||
@@ -529,7 +523,6 @@ def minimax_m3_sparse_attn_decode(
|
||||
kv_cache.stride(1),
|
||||
kv_cache.stride(2),
|
||||
kv_cache.stride(3),
|
||||
kv_cache.stride(4),
|
||||
topk_idx.stride(0),
|
||||
topk_idx.stride(1),
|
||||
topk_idx.stride(2),
|
||||
|
||||
@@ -104,20 +104,25 @@ class MiniMaxM3SparseBackend(AttentionBackend):
|
||||
head_size: int,
|
||||
cache_dtype_str: str = "auto",
|
||||
) -> tuple[int, ...]:
|
||||
return (num_blocks, 2, block_size, num_kv_heads, head_size)
|
||||
# K and V are packed into the content dim: logical (B, H, N, 2*hs).
|
||||
return (num_blocks, num_kv_heads, block_size, 2 * head_size)
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_stride_order(
|
||||
include_num_layers_dimension: bool = False,
|
||||
) -> tuple[int, ...]:
|
||||
# Permutation from get_kv_cache_shape to the actual memory layout.
|
||||
# `stride_order` indicates the permutation that gets us from
|
||||
# `get_kv_cache_shape` (logical (B, H, N, 2*hs)) to the actual memory
|
||||
# layout we want.
|
||||
if include_num_layers_dimension:
|
||||
raise NotImplementedError # no cross-layer KV blocks in M3
|
||||
cache_layout = get_kv_cache_layout()
|
||||
if cache_layout == "NHD":
|
||||
stride_order = (0, 1, 2, 3, 4)
|
||||
# (num_blocks, block_size, num_kv_heads, 2*head_size)
|
||||
stride_order = (0, 2, 1, 3)
|
||||
elif cache_layout == "HND":
|
||||
stride_order = (0, 1, 3, 2, 4)
|
||||
# (num_blocks, num_kv_heads, block_size, 2*head_size)
|
||||
stride_order = (0, 1, 2, 3)
|
||||
else:
|
||||
raise ValueError(f"Unknown cache layout format {cache_layout}.")
|
||||
return stride_order
|
||||
|
||||
@@ -79,8 +79,7 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
|
||||
# strided view directly (topK stays innermost-contiguous).
|
||||
prefill_topk = topk[nd:num_tokens].transpose(0, 1)
|
||||
qp = q[nd:]
|
||||
k_cache = kv_cache[:, 0].transpose(1, 2)
|
||||
v_cache = kv_cache[:, 1].transpose(1, 2)
|
||||
k_cache, v_cache = kv_cache.split(self.head_size, dim=-1)
|
||||
k2q_row_ptr, k2q_q_indices, schedule = build_k2q_csr(
|
||||
prefill_topk,
|
||||
p.cu_seqlens_q,
|
||||
|
||||
+16
-2
@@ -427,8 +427,8 @@ def _get_backend_priorities(
|
||||
]
|
||||
|
||||
backends = []
|
||||
# ROCM_ATTN uses (2, num_blocks, ...) KV cache layout which is
|
||||
# incompatible with KV connectors that require blocks-first layout.
|
||||
# Keep ROCM_ATTN disabled for KV connectors until connector transfer
|
||||
# semantics are validated for its asymmetric native K/V cache views.
|
||||
if not use_kv_connector:
|
||||
backends.append(AttentionBackendEnum.ROCM_ATTN)
|
||||
if rocm_aiter_ops.is_mha_enabled():
|
||||
@@ -511,6 +511,20 @@ class RocmPlatform(Platform):
|
||||
attn_selector_config.use_sparse,
|
||||
attn_selector_config.use_kv_connector,
|
||||
)
|
||||
from vllm.config import get_current_vllm_config_or_none
|
||||
|
||||
vllm_config = get_current_vllm_config_or_none()
|
||||
is_encoder_decoder = (
|
||||
getattr(getattr(vllm_config, "model_config", None), "attn_type", None)
|
||||
== "encoder_decoder"
|
||||
)
|
||||
# ROCM_ATTN still uses a legacy attention layout (KV is the outer
|
||||
# dimension) that is incompatible with the encoder backend layouts. The
|
||||
# encoder and decoder need the layouts to match. This is currently
|
||||
# enforced implicitly.
|
||||
# TODO: Make this explicit in the selector in a future PR.
|
||||
if is_encoder_decoder and AttentionBackendEnum.ROCM_ATTN in backend_priorities:
|
||||
backend_priorities.remove(AttentionBackendEnum.ROCM_ATTN)
|
||||
for priority, backend in enumerate(backend_priorities):
|
||||
try:
|
||||
backend_class = backend.get_class()
|
||||
|
||||
+11
-45
@@ -416,26 +416,24 @@ def nvfp4_kv_cache_full_dim(head_size: int) -> int:
|
||||
return head_size // 2 + head_size // 16
|
||||
|
||||
|
||||
def _nvfp4_split_data_scale(
|
||||
def nvfp4_split_data_scale(
|
||||
kv_side: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Split a single NVFP4 KV-side buffer into data and scale views.
|
||||
"""Split one side (K or V) of an NVFP4 KV cache into data and scale.
|
||||
|
||||
The input is a 4D tensor for one KV side (K or V) whose last
|
||||
dimension is ``full_dim = data_dim + scale_dim``. The physical
|
||||
layout within each side is [data | scale], both packed contiguously.
|
||||
The input is a 4D uint8 tensor whose last dimension is
|
||||
``full_dim = data_dim + scale_dim``. The physical layout within each
|
||||
side is ``[data | scale]``, both packed contiguously.
|
||||
|
||||
The caller is responsible for slicing K and V from the combined cache
|
||||
first (e.g. ``kv_cache.split(num_kv_heads, dim=1)``).
|
||||
|
||||
Args:
|
||||
kv_side: 4D uint8 tensor with shape
|
||||
``(num_pages, dim_1, dim_2, full_dim)``.
|
||||
May be in any permutation order (NHD or HND).
|
||||
kv_side: 4D uint8 tensor ``(B, H, N, full_dim)``.
|
||||
|
||||
Returns:
|
||||
``(data, scale)`` where
|
||||
``data`` is a uint8 view with shape
|
||||
``(num_pages, dim_1, dim_2, data_dim)``.
|
||||
``scale`` is a float8_e4m3fn view with shape
|
||||
``(num_pages, dim_1, dim_2, scale_dim)``.
|
||||
``(data, scale)`` where *data* is uint8 and *scale* is
|
||||
float8_e4m3fn, both views of the same storage.
|
||||
"""
|
||||
num_pages = kv_side.shape[0]
|
||||
dim_1, dim_2 = kv_side.shape[1], kv_side.shape[2]
|
||||
@@ -468,38 +466,6 @@ def _nvfp4_split_data_scale(
|
||||
return data, scale
|
||||
|
||||
|
||||
def nvfp4_kv_cache_split_views(kv_cache: torch.Tensor) -> tuple[tuple, tuple]:
|
||||
"""Split an NVFP4 KV cache tensor into data and scale views.
|
||||
|
||||
Accepts either a 5D tensor ``(num_pages, 2, dim_2, dim_3, full_dim)``
|
||||
or a 4D single-side tensor ``(num_pages, dim_2, dim_3, full_dim)``.
|
||||
|
||||
Per-page layout: [K_data | K_scale | V_data | V_scale].
|
||||
Each KV side is self-contained (data followed by its scale), so the
|
||||
5D case simply splits each side independently.
|
||||
|
||||
The returned views are in the same dim order as the input (NHD or
|
||||
HND), so callers get views matching whichever order they passed in.
|
||||
|
||||
Args:
|
||||
kv_cache: 5D or 4D uint8 tensor where the last dimension is
|
||||
``full_dim = data_dim + scale_dim = 9 * head_size / 16``.
|
||||
|
||||
Returns:
|
||||
For 5D input:
|
||||
``(k_data, v_data), (k_scale, v_scale)``
|
||||
For 4D input (single KV side):
|
||||
``(data,), (scale,)``
|
||||
"""
|
||||
if kv_cache.dim() == 4:
|
||||
data, scale = _nvfp4_split_data_scale(kv_cache)
|
||||
return (data,), (scale,)
|
||||
|
||||
k_data, k_scale = _nvfp4_split_data_scale(kv_cache[:, 0])
|
||||
v_data, v_scale = _nvfp4_split_data_scale(kv_cache[:, 1])
|
||||
return (k_data, v_data), (k_scale, v_scale)
|
||||
|
||||
|
||||
def create_kv_caches_with_random_flash(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
|
||||
@@ -122,11 +122,11 @@ class AttentionBackend(ABC):
|
||||
) -> tuple[int, ...]:
|
||||
"""
|
||||
Get the physical (memory layout) ordering of the kv cache dimensions.
|
||||
e.g. if the KV cache shape is
|
||||
[2, num_blocks, block_size, num_heads, head_size],
|
||||
and get_kv_cache_stride_order returns (1, 3, 0, 2, 4) then the physical
|
||||
Standard attention backends pack K and V into the content dim, giving
|
||||
the logical shape [num_blocks, num_heads, block_size, 2 * head_size].
|
||||
e.g. if get_kv_cache_stride_order returns (0, 2, 1, 3) then the physical
|
||||
ordering of dimensions is
|
||||
[num_blocks, num_heads, 2, block_size, head_size].
|
||||
[num_blocks, block_size, num_heads, 2 * head_size].
|
||||
|
||||
If this function is unimplemented / raises NotImplementedError,
|
||||
the physical layout of the KV cache will match the logical shape.
|
||||
@@ -135,9 +135,9 @@ class AttentionBackend(ABC):
|
||||
include_num_layers_dimension: if True, includes an additional
|
||||
num_layers dimension, which is assumed to be prepended
|
||||
to the logical KV cache shape.
|
||||
With the above example, a return value (2, 4, 0, 1, 3, 5)
|
||||
With the above example, a return value (1, 0, 3, 2, 4)
|
||||
corresponds to
|
||||
[num_blocks, num_heads, num_layers, 2, block_size, head_size].
|
||||
[num_blocks, num_layers, block_size, num_heads, 2 * head_size].
|
||||
|
||||
If an additional dimension is NOT included in the returned
|
||||
tuple, the physical layout will not include a layers dimension.
|
||||
|
||||
@@ -133,25 +133,29 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
) -> tuple[int, ...]:
|
||||
if block_size % 16 != 0:
|
||||
raise ValueError("Block size must be a multiple of 16.")
|
||||
return (num_blocks, 2, block_size, num_kv_heads, head_size)
|
||||
# K and V are packed into the content dim: logical (B, H, N, 2*D).
|
||||
return (num_blocks, num_kv_heads, block_size, 2 * head_size)
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_stride_order(
|
||||
include_num_layers_dimension: bool = False,
|
||||
) -> tuple[int, ...]:
|
||||
# `stride_order` indicates the permutation that gets
|
||||
# us from `get_kv_cache_shape` to the actual memory layout we want.
|
||||
# `stride_order` indicates the permutation that gets us from
|
||||
# `get_kv_cache_shape` (logical (B, H, N, 2*D)) to the actual memory
|
||||
# layout we want.
|
||||
cache_layout = get_kv_cache_layout()
|
||||
if cache_layout == "NHD" and include_num_layers_dimension:
|
||||
# (num_blocks, num_layers, 2, block_size, num_kv_heads, head_size)
|
||||
return (1, 0, 2, 3, 4, 5)
|
||||
# (num_blocks, num_layers, block_size, num_kv_heads, 2*head_size)
|
||||
return (1, 0, 3, 2, 4)
|
||||
elif cache_layout == "NHD":
|
||||
stride_order = (0, 1, 2, 3, 4)
|
||||
# (num_blocks, block_size, num_kv_heads, 2*head_size)
|
||||
stride_order = (0, 2, 1, 3)
|
||||
elif cache_layout == "HND" and include_num_layers_dimension:
|
||||
# (num_blocks, num_kv_heads, num_layers, 2, block_size, head_size)
|
||||
return (1, 4, 0, 2, 3, 5)
|
||||
# (num_blocks, num_kv_heads, num_layers, block_size, 2*head_size)
|
||||
return (1, 2, 0, 3, 4)
|
||||
elif cache_layout == "HND":
|
||||
stride_order = (0, 1, 3, 2, 4)
|
||||
# (num_blocks, num_kv_heads, block_size, 2*head_size)
|
||||
stride_order = (0, 1, 2, 3)
|
||||
else:
|
||||
raise ValueError(f"Unknown cache layout format {cache_layout}.")
|
||||
return stride_order
|
||||
@@ -819,7 +823,7 @@ class FlashAttentionImpl(AttentionImpl):
|
||||
key: shape = [num_tokens, num_kv_heads, head_size]
|
||||
value: shape = [num_tokens, num_kv_heads, head_size]
|
||||
kv_cache: shape =
|
||||
[num_blocks, 2, block_size, num_kv_heads, head_size]
|
||||
[num_blocks, num_kv_heads, block_size, 2 * head_size]
|
||||
attn_metadata: Metadata for attention.
|
||||
Returns:
|
||||
shape = [num_tokens, num_heads * head_size]
|
||||
@@ -866,8 +870,8 @@ class FlashAttentionImpl(AttentionImpl):
|
||||
layer,
|
||||
)
|
||||
|
||||
# For decoder and cross-attention, use KV cache as before
|
||||
key_cache, value_cache = kv_cache.unbind(1)
|
||||
# (B, H, N, 2*D) -> ((B, N, H, D), (B, N, H, D))
|
||||
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
|
||||
# Fix degenerate strides on size-1 dims (e.g. num_kv_heads=1 with TP).
|
||||
# FA3/4 on H100+ uses TMA, which requires ≥16-byte stride alignment.
|
||||
# See vllm.utils.torch_utils.canonicalize_singleton_dim_strides.
|
||||
@@ -1075,7 +1079,8 @@ class FlashAttentionImpl(AttentionImpl):
|
||||
|
||||
# Scatter write into the KV cache using slot_mapping indices.
|
||||
# No TMA kernel is invoked here, so stride canonicalization is not needed.
|
||||
key_cache, value_cache = kv_cache.unbind(1)
|
||||
# (B, H, N, 2*D) -> ((B, N, H, D), (B, N, H, D))
|
||||
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
|
||||
|
||||
# Reshape the input keys and values and store them in the cache.
|
||||
# Skip this if sharing KV cache with an earlier attention layer.
|
||||
|
||||
@@ -83,10 +83,12 @@ class FlashAttentionDiffKVBackend(FlashAttentionBackend):
|
||||
) -> tuple[int, ...]:
|
||||
if block_size % 16 != 0:
|
||||
raise ValueError("Block size must be a multiple of 16.")
|
||||
# Logical (blocks-first, head-major) layout: K and V (with their
|
||||
# different head sizes) packed in the content dim.
|
||||
return (
|
||||
num_blocks,
|
||||
block_size,
|
||||
num_kv_heads,
|
||||
block_size,
|
||||
head_size + FlashAttentionDiffKVBackend.head_size_v,
|
||||
)
|
||||
|
||||
@@ -94,21 +96,22 @@ class FlashAttentionDiffKVBackend(FlashAttentionBackend):
|
||||
def get_kv_cache_stride_order(
|
||||
include_num_layers_dimension: bool = False,
|
||||
) -> tuple[int, ...]:
|
||||
# `stride_order` indicates the permutation that gets
|
||||
# us from `get_kv_cache_shape` to the actual memory layout we want.
|
||||
# `stride_order` indicates the permutation that gets us from
|
||||
# `get_kv_cache_shape` (logical (B, H, N, C_k+C_v)) to the actual
|
||||
# memory layout we want.
|
||||
cache_layout = get_kv_cache_layout()
|
||||
if cache_layout == "NHD" and include_num_layers_dimension:
|
||||
# (num_blocks, num_layers, block_size,
|
||||
# num_kv_heads, head_size + head_size_v)
|
||||
return (1, 0, 2, 3, 4)
|
||||
# (num_blocks, num_layers, block_size, num_kv_heads, C_k+C_v)
|
||||
return (1, 0, 3, 2, 4)
|
||||
elif cache_layout == "NHD":
|
||||
stride_order = (0, 1, 2, 3)
|
||||
elif cache_layout == "HND" and include_num_layers_dimension:
|
||||
# (num_blocks, num_kv_heads, num_layers,
|
||||
# block_size, head_size + head_size_v)
|
||||
return (1, 3, 0, 2, 4)
|
||||
elif cache_layout == "HND":
|
||||
# (num_blocks, block_size, num_kv_heads, C_k+C_v)
|
||||
stride_order = (0, 2, 1, 3)
|
||||
elif cache_layout == "HND" and include_num_layers_dimension:
|
||||
# (num_blocks, num_kv_heads, num_layers, block_size, C_k+C_v)
|
||||
return (1, 2, 0, 3, 4)
|
||||
elif cache_layout == "HND":
|
||||
# (num_blocks, num_kv_heads, block_size, C_k+C_v)
|
||||
stride_order = (0, 1, 2, 3)
|
||||
else:
|
||||
raise ValueError(f"Unknown cache layout format {cache_layout}.")
|
||||
return stride_order
|
||||
@@ -142,21 +145,14 @@ class FlashAttentionDiffKVImpl(FlashAttentionImpl):
|
||||
# we use direct Q, K, V tensors without caching
|
||||
return
|
||||
|
||||
# Unlike standard FlashAttn which splits kv_cache via unbind(0),
|
||||
# DiffKV packs K and V into a single tensor along the last dim:
|
||||
# kv_cache shape: [num_blocks, block_size, num_kv_heads,
|
||||
# head_size_k + head_size_v]
|
||||
# The triton kernel handles this combined layout directly.
|
||||
#
|
||||
# NOTE(woosuk): key and value are padded while slot_mapping is
|
||||
# not padded. However, we don't need to do key[:num_actual_tokens]
|
||||
# and value[:num_actual_tokens] because the reshape_and_cache_flash
|
||||
# op uses the slot_mapping's shape to determine the number of
|
||||
# actual tokens.
|
||||
# (B, H, N, C) -> (B, N, H, C) for kernel compatibility.
|
||||
triton_reshape_and_cache_flash_diffkv(
|
||||
key,
|
||||
value,
|
||||
kv_cache,
|
||||
kv_cache.transpose(1, 2),
|
||||
slot_mapping,
|
||||
self.kv_cache_dtype,
|
||||
layer._k_scale,
|
||||
@@ -229,10 +225,8 @@ class FlashAttentionDiffKVImpl(FlashAttentionImpl):
|
||||
layer,
|
||||
)
|
||||
|
||||
# For decoder and cross-attention, use KV cache as before
|
||||
# Different head_size for K and V
|
||||
key_cache = kv_cache[..., : self.head_size]
|
||||
value_cache = kv_cache[..., self.head_size :]
|
||||
# (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs))
|
||||
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
|
||||
# Fix degenerate strides on size-1 dims (e.g. num_kv_heads=1 with TP).
|
||||
# FA3/4 on H100+ uses TMA, which requires ≥16-byte stride alignment.
|
||||
# See vllm.utils.torch_utils.canonicalize_singleton_dim_strides.
|
||||
|
||||
@@ -51,7 +51,7 @@ from vllm.utils.torch_utils import (
|
||||
is_quantized_kv_cache,
|
||||
is_strictly_contiguous,
|
||||
nvfp4_kv_cache_full_dim,
|
||||
nvfp4_kv_cache_split_views,
|
||||
nvfp4_split_data_scale,
|
||||
)
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
@@ -111,9 +111,12 @@ def _trtllm_prefill_attn_kvfp8_dequant(
|
||||
src_stride_page,
|
||||
src_stride_kv,
|
||||
src_stride_head,
|
||||
src_stride_block,
|
||||
src_stride_head_size,
|
||||
DST_K_CACHE_STRIDE: tl.constexpr,
|
||||
DST_KV_CACHE_STRIDE: tl.constexpr,
|
||||
HEAD_STRIDE: tl.constexpr,
|
||||
HEAD_SIZE: tl.constexpr,
|
||||
NUM_KV_HEADS: tl.constexpr,
|
||||
):
|
||||
batch_idx = tl.program_id(0).to(tl.int64)
|
||||
@@ -129,18 +132,25 @@ def _trtllm_prefill_attn_kvfp8_dequant(
|
||||
v_scale_val = tl.load(v_scale_ptr)
|
||||
|
||||
mock_page_idx = batch_idx * block_table_stride + mock_block_table_idx + 1
|
||||
head_offsets = tl.arange(0, HEAD_STRIDE)
|
||||
logical_offsets = tl.arange(0, HEAD_STRIDE)
|
||||
block_offsets = logical_offsets // HEAD_SIZE
|
||||
head_size_offsets = logical_offsets % HEAD_SIZE
|
||||
|
||||
for h in range(NUM_KV_HEADS):
|
||||
h_off = tl.cast(h, tl.int64)
|
||||
|
||||
# Read K from source (supports non-contiguous page/kv/head strides)
|
||||
src_k = orig_page_num * src_stride_page + h_off * src_stride_head + head_offsets
|
||||
src_k = (
|
||||
orig_page_num * src_stride_page
|
||||
+ h_off * src_stride_head
|
||||
+ block_offsets * src_stride_block
|
||||
+ head_size_offsets * src_stride_head_size
|
||||
)
|
||||
fp8_k = tl.load(kv_cache_ptr + src_k)
|
||||
dequant_k = (fp8_k.to(tl.float32) * k_scale_val).to(dequant_dtype)
|
||||
|
||||
# Write K to contiguous mock cache
|
||||
dst_k = mock_page_idx * DST_KV_CACHE_STRIDE + h * HEAD_STRIDE + head_offsets
|
||||
dst_k = mock_page_idx * DST_KV_CACHE_STRIDE + h * HEAD_STRIDE + logical_offsets
|
||||
tl.store(mock_kv_cache_ptr + dst_k, dequant_k)
|
||||
|
||||
# Read V from source (offset by src_stride_kv for the V half)
|
||||
@@ -148,7 +158,8 @@ def _trtllm_prefill_attn_kvfp8_dequant(
|
||||
orig_page_num * src_stride_page
|
||||
+ src_stride_kv
|
||||
+ h_off * src_stride_head
|
||||
+ head_offsets
|
||||
+ block_offsets * src_stride_block
|
||||
+ head_size_offsets * src_stride_head_size
|
||||
)
|
||||
fp8_v = tl.load(kv_cache_ptr + src_v)
|
||||
dequant_v = (fp8_v.to(tl.float32) * v_scale_val).to(dequant_dtype)
|
||||
@@ -158,7 +169,7 @@ def _trtllm_prefill_attn_kvfp8_dequant(
|
||||
mock_page_idx * DST_KV_CACHE_STRIDE
|
||||
+ DST_K_CACHE_STRIDE
|
||||
+ h * HEAD_STRIDE
|
||||
+ head_offsets
|
||||
+ logical_offsets
|
||||
)
|
||||
tl.store(mock_kv_cache_ptr + dst_v, dequant_v)
|
||||
|
||||
@@ -175,17 +186,14 @@ def trtllm_prefill_attn_kvfp8_dequant(
|
||||
assert s[1] == 2
|
||||
assert dequant_dtype in (torch.bfloat16, torch.float16)
|
||||
|
||||
# Logical source layout is (B, 2, H, N, D). The tensor may be a
|
||||
# non-contiguous view, so the Triton kernel indexes it with actual strides.
|
||||
strides = kv_cache.stride()
|
||||
num_kv_heads, block_size, head_size = s[2], s[3], s[4]
|
||||
head_stride = block_size * head_size
|
||||
k_cache_stride = num_kv_heads * head_stride
|
||||
kv_cache_stride = k_cache_stride * s[1]
|
||||
|
||||
strides = kv_cache.stride()
|
||||
assert strides[3] == head_size and strides[4] == 1, (
|
||||
"For kv cache layouts, (block_size, head_size) "
|
||||
f"dimensions must be contiguous, got strides {strides}"
|
||||
)
|
||||
|
||||
new_s = (batch_size * num_of_page_per_token + 1, s[1], s[2], s[3], s[4])
|
||||
# mock kv cache contains just the pages needed by this prefill
|
||||
mock_kv_cache = torch.empty(new_s, dtype=dequant_dtype, device=kv_cache.device)
|
||||
@@ -207,9 +215,12 @@ def trtllm_prefill_attn_kvfp8_dequant(
|
||||
strides[0],
|
||||
strides[1],
|
||||
strides[2],
|
||||
strides[3],
|
||||
strides[4],
|
||||
k_cache_stride,
|
||||
kv_cache_stride,
|
||||
head_stride,
|
||||
head_size,
|
||||
num_kv_heads,
|
||||
)
|
||||
return mock_kv_cache, mock_block_table
|
||||
@@ -286,7 +297,7 @@ class BatchDCPPrefillWrapper:
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
prefill_query: torch.Tensor,
|
||||
kv_cache_permute: torch.Tensor,
|
||||
kv_cache_tuple: tuple[torch.Tensor, torch.Tensor],
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
@@ -296,7 +307,7 @@ class BatchDCPPrefillWrapper:
|
||||
)
|
||||
output_context_tmp, lse_context_tmp = self._context.run(
|
||||
prefill_query_across_dcp,
|
||||
kv_cache_permute,
|
||||
kv_cache_tuple,
|
||||
k_scale=layer._k_scale_float,
|
||||
v_scale=layer._v_scale_float,
|
||||
return_lse=True,
|
||||
@@ -386,28 +397,31 @@ class FlashInferBackend(AttentionBackend):
|
||||
cache_dtype_str: str = "auto",
|
||||
) -> tuple[int, ...]:
|
||||
if cache_dtype_str == "nvfp4":
|
||||
# Packed layout: fp4 data + fp8 block scales in last dim
|
||||
last_dim = nvfp4_kv_cache_full_dim(head_size)
|
||||
return (num_blocks, 2, block_size, num_kv_heads, last_dim)
|
||||
return (num_blocks, 2, block_size, num_kv_heads, head_size)
|
||||
full_dim = nvfp4_kv_cache_full_dim(head_size)
|
||||
return (num_blocks, 2 * num_kv_heads, block_size, full_dim)
|
||||
# Pack K and V in the content dim (B, H, N, 2*hs).
|
||||
return (num_blocks, num_kv_heads, block_size, 2 * head_size)
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_stride_order(
|
||||
include_num_layers_dimension: bool = False,
|
||||
) -> tuple[int, ...]:
|
||||
# `stride_order` indicates the permutation that gets us from
|
||||
# `get_kv_cache_shape` to the actual memory layout we want.
|
||||
# `get_kv_cache_shape` (logical (B, H, N, 2*hs)) to the actual memory
|
||||
# layout we want.
|
||||
cache_layout = get_kv_cache_layout()
|
||||
if cache_layout == "NHD" and include_num_layers_dimension:
|
||||
# (num_blocks, num_layers, 2, block_size, num_kv_heads, head_size)
|
||||
return (1, 0, 2, 3, 4, 5)
|
||||
# (num_blocks, num_layers, block_size, num_kv_heads, 2*head_size)
|
||||
return (1, 0, 3, 2, 4)
|
||||
elif cache_layout == "NHD":
|
||||
stride_order = (0, 1, 2, 3, 4)
|
||||
# (num_blocks, block_size, num_kv_heads, 2*head_size)
|
||||
stride_order = (0, 2, 1, 3)
|
||||
elif cache_layout == "HND" and include_num_layers_dimension:
|
||||
# (num_blocks, 2, num_kv_heads, num_layers, block_size, head_size)
|
||||
return (1, 2, 4, 0, 3, 5)
|
||||
# (num_blocks, num_kv_heads, num_layers, block_size, 2*head_size)
|
||||
return (1, 2, 0, 3, 4)
|
||||
elif cache_layout == "HND":
|
||||
stride_order = (0, 1, 3, 2, 4)
|
||||
# (num_blocks, num_kv_heads, block_size, 2*head_size)
|
||||
stride_order = (0, 1, 2, 3)
|
||||
else:
|
||||
raise ValueError(f"Unknown cache layout format {cache_layout}.")
|
||||
return stride_order
|
||||
@@ -1714,7 +1728,19 @@ class FlashInferImpl(AttentionImpl):
|
||||
if attn_metadata.use_cascade:
|
||||
# Cascade attention (rare case).
|
||||
assert attn_metadata.cascade_wrapper is not None
|
||||
output.copy_(attn_metadata.cascade_wrapper.run(query, kv_cache))
|
||||
stride_order = FlashInferBackend.get_kv_cache_stride_order()
|
||||
if self.is_kvcache_nvfp4:
|
||||
kv_cache_views = tuple(
|
||||
cache.permute(*stride_order)
|
||||
for cache in kv_cache.split(self.num_kv_heads, dim=1)
|
||||
)
|
||||
else:
|
||||
kv_perm = kv_cache.permute(*stride_order)
|
||||
kv_cache_views = kv_perm.split(self.head_size, dim=-1)
|
||||
kv_tuple = tuple(
|
||||
canonicalize_singleton_dim_strides(cache) for cache in kv_cache_views
|
||||
)
|
||||
output.copy_(attn_metadata.cascade_wrapper.run(query, kv_tuple))
|
||||
return output
|
||||
|
||||
# When using spec decoding, num_decodes can be < num_decode_tokens
|
||||
@@ -1740,14 +1766,23 @@ class FlashInferImpl(AttentionImpl):
|
||||
)
|
||||
kv_cache_permute = fixed
|
||||
|
||||
# For NVFP4, the kv_cache last dim is full_dim (data + scale packed).
|
||||
# Split into correctly-strided data and scale views.
|
||||
# Split K/V — zero-copy views. NVFP4 stores K/V as separate head
|
||||
# groups; other dtypes pack K/V in the content dim.
|
||||
hs = self.head_size
|
||||
nvfp4_kv_data = None
|
||||
nvfp4_kv_block_scales = None
|
||||
if self.is_kvcache_nvfp4:
|
||||
nvfp4_kv_data, nvfp4_kv_block_scales = nvfp4_kv_cache_split_views(
|
||||
kv_cache_permute
|
||||
k_cache, v_cache = kv_cache.split(self.num_kv_heads, dim=1)
|
||||
kv_cache_tuple = (
|
||||
canonicalize_singleton_dim_strides(k_cache.permute(*stride_order)),
|
||||
canonicalize_singleton_dim_strides(v_cache.permute(*stride_order)),
|
||||
)
|
||||
k_data, k_sf = nvfp4_split_data_scale(kv_cache_tuple[0])
|
||||
v_data, v_sf = nvfp4_split_data_scale(kv_cache_tuple[1])
|
||||
nvfp4_kv_data = (k_data, v_data)
|
||||
nvfp4_kv_block_scales = (k_sf, v_sf)
|
||||
else:
|
||||
kv_cache_tuple = kv_cache_permute.split(hs, dim=-1)
|
||||
|
||||
use_dcp = self.dcp_world_size > 1
|
||||
|
||||
@@ -1786,7 +1821,7 @@ class FlashInferImpl(AttentionImpl):
|
||||
prefill_wrapper.run(
|
||||
layer,
|
||||
prefill_query,
|
||||
kv_cache_permute,
|
||||
kv_cache_tuple,
|
||||
key[num_decode_tokens:],
|
||||
value[num_decode_tokens:],
|
||||
out=output[num_decode_tokens:],
|
||||
@@ -1803,7 +1838,9 @@ class FlashInferImpl(AttentionImpl):
|
||||
assert prefill_wrapper._causal == attn_metadata.causal
|
||||
|
||||
if self.is_kvcache_nvfp4:
|
||||
kv_cache_permute = nvfp4_kv_data
|
||||
kv_cache_for_fi = nvfp4_kv_data
|
||||
else:
|
||||
kv_cache_for_fi = kv_cache_tuple
|
||||
kv_cache_sf = (
|
||||
nvfp4_kv_block_scales if self.is_kvcache_nvfp4 else None
|
||||
)
|
||||
@@ -1821,7 +1858,7 @@ class FlashInferImpl(AttentionImpl):
|
||||
|
||||
prefill_wrapper.run(
|
||||
prefill_query,
|
||||
kv_cache_permute,
|
||||
kv_cache_for_fi,
|
||||
q_scale=layer._q_scale_float,
|
||||
k_scale=layer._k_scale_float,
|
||||
v_scale=layer._v_scale_float,
|
||||
@@ -1888,8 +1925,7 @@ class FlashInferImpl(AttentionImpl):
|
||||
# TRTLLM prefill attention does not support BF16 Q
|
||||
# and fp8 kv cache. So to enable prefill attention
|
||||
# with fp8 kv cache, we can construct a mock block
|
||||
# and mock kv cache with BF16 KV involved in the prefill
|
||||
#
|
||||
# and mock kv cache with BF16 KV involved in the prefill.
|
||||
kv_cache_permute = canonicalize_singleton_dim_strides(
|
||||
kv_cache_permute
|
||||
)
|
||||
@@ -1901,15 +1937,21 @@ class FlashInferImpl(AttentionImpl):
|
||||
"KV cache inner dims (block_size, head_size) must be "
|
||||
f"contiguous, got strides {kv_strides}"
|
||||
)
|
||||
# fp8 uses (B, H, N, 2*hs); reshape to (B, 2, H, N, hs)
|
||||
# for the dequant kernel — zero-copy view. The dequant
|
||||
# kernel handles the interleaved K/V block stride.
|
||||
B_kv, H_kv, N_kv = kv_cache_permute.shape[:3]
|
||||
kv_cache_5d = kv_cache_permute.view(B_kv, H_kv, N_kv, 2, hs)
|
||||
kv_cache_5d = kv_cache_5d.permute(0, 3, 1, 2, 4)
|
||||
mock_kv_cache, mock_block_table = trtllm_prefill_attn_kvfp8_dequant(
|
||||
kv_cache_permute,
|
||||
kv_cache_5d,
|
||||
block_tables_prefill,
|
||||
layer._k_scale,
|
||||
layer._v_scale,
|
||||
attn_metadata.q_data_type_prefill,
|
||||
)
|
||||
else:
|
||||
mock_kv_cache = kv_cache_permute
|
||||
mock_kv_cache = kv_cache_tuple
|
||||
mock_block_table = block_tables_prefill
|
||||
|
||||
trtllm_batch_context_with_kv_cache(
|
||||
@@ -1957,7 +1999,9 @@ class FlashInferImpl(AttentionImpl):
|
||||
assert decode_wrapper._sm_scale == self.scale
|
||||
|
||||
if self.is_kvcache_nvfp4:
|
||||
kv_cache_permute = nvfp4_kv_data
|
||||
kv_cache_for_fi = nvfp4_kv_data
|
||||
else:
|
||||
kv_cache_for_fi = kv_cache_tuple
|
||||
kv_cache_sf = nvfp4_kv_block_scales if self.is_kvcache_nvfp4 else None
|
||||
|
||||
# NVFP4 kernel only supports FP8 output.
|
||||
@@ -1980,7 +2024,7 @@ class FlashInferImpl(AttentionImpl):
|
||||
)
|
||||
decode_wrapper.run(
|
||||
decode_query,
|
||||
kv_cache_permute,
|
||||
kv_cache_for_fi,
|
||||
q_scale=layer._q_scale_float,
|
||||
k_scale=layer._k_scale_float,
|
||||
v_scale=layer._v_scale_float,
|
||||
@@ -1997,7 +2041,7 @@ class FlashInferImpl(AttentionImpl):
|
||||
else:
|
||||
decode_wrapper.run(
|
||||
decode_query,
|
||||
kv_cache_permute,
|
||||
kv_cache_for_fi,
|
||||
q_scale=layer._q_scale_float,
|
||||
k_scale=layer._k_scale_float,
|
||||
v_scale=layer._v_scale_float,
|
||||
@@ -2079,7 +2123,7 @@ class FlashInferImpl(AttentionImpl):
|
||||
trtllm_batch_decode_with_kv_cache(
|
||||
query=decode_query,
|
||||
kv_cache=(
|
||||
nvfp4_kv_data if self.is_kvcache_nvfp4 else kv_cache_permute
|
||||
nvfp4_kv_data if self.is_kvcache_nvfp4 else kv_cache_tuple
|
||||
),
|
||||
workspace_buffer=workspace_buffer,
|
||||
block_tables=block_tables_decode,
|
||||
@@ -2119,8 +2163,18 @@ class FlashInferImpl(AttentionImpl):
|
||||
# and value[:num_actual_tokens] because the reshape_and_cache_flash
|
||||
# op uses the slot_mapping's shape to determine the number of
|
||||
# actual tokens.
|
||||
k_cache = kv_cache[:, 0]
|
||||
v_cache = kv_cache[:, 1]
|
||||
if self.is_kvcache_nvfp4:
|
||||
# (B, 2*H, N, full_dim) -> ((B, N, H, full_dim),
|
||||
# (B, N, H, full_dim));
|
||||
# K heads first, then V heads.
|
||||
k_cache, v_cache = kv_cache.transpose(1, 2).split(
|
||||
self.num_kv_heads, dim=-2
|
||||
)
|
||||
else:
|
||||
# (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs))
|
||||
k_cache, v_cache = kv_cache.transpose(1, 2).split(
|
||||
self.head_size, dim=-1
|
||||
)
|
||||
torch.ops._C_cache_ops.reshape_and_cache_flash(
|
||||
key,
|
||||
value,
|
||||
|
||||
@@ -130,15 +130,16 @@ class FlexAttentionBackend(AttentionBackend):
|
||||
head_size: int,
|
||||
cache_dtype_str: str = "auto",
|
||||
) -> tuple[int, ...]:
|
||||
return (num_blocks, 2, block_size, num_kv_heads, head_size)
|
||||
# K and V are packed into the content dim: logical (B, H, N, 2*hs).
|
||||
return (num_blocks, num_kv_heads, block_size, 2 * head_size)
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_stride_order(
|
||||
include_num_layers_dimension: bool = False,
|
||||
) -> tuple[int, ...]:
|
||||
if include_num_layers_dimension:
|
||||
return (1, 0, 3, 2, 4, 5)
|
||||
return (0, 2, 1, 3, 4)
|
||||
return (1, 0, 3, 2, 4)
|
||||
return (0, 2, 1, 3)
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type["FlexAttentionMetadataBuilder"]:
|
||||
@@ -1242,7 +1243,8 @@ class FlexAttentionImpl(AttentionImpl):
|
||||
if self.attn_type == AttentionType.ENCODER_ONLY:
|
||||
return
|
||||
|
||||
key_cache, value_cache = kv_cache.unbind(1)
|
||||
# (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs))
|
||||
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
|
||||
torch.ops._C_cache_ops.reshape_and_cache_flash(
|
||||
key,
|
||||
value,
|
||||
@@ -1273,7 +1275,7 @@ class FlexAttentionImpl(AttentionImpl):
|
||||
key: shape = [num_tokens, num_kv_heads, head_size]
|
||||
value: shape = [num_tokens, num_kv_heads, head_size]
|
||||
kv_cache: shape =
|
||||
[num_blocks, 2, block_size, num_kv_heads, head_size]
|
||||
[num_blocks, num_kv_heads, block_size, 2 * head_size]
|
||||
attn_metadata: Metadata for attention.
|
||||
Returns:
|
||||
shape = [num_tokens, num_heads * head_size]
|
||||
@@ -1347,9 +1349,11 @@ class FlexAttentionImpl(AttentionImpl):
|
||||
|
||||
else:
|
||||
assert self.attn_type == AttentionType.DECODER
|
||||
key_cache, value_cache = kv_cache.unbind(1)
|
||||
kv_cache = kv_cache.transpose(1, 2)
|
||||
hs = self.head_size
|
||||
key_cache, value_cache = kv_cache.split(hs, dim=-1)
|
||||
|
||||
# Flatten (num_blocks, block_size) into a single token dim
|
||||
# Flatten (num_blocks, block_size) into a single token dim.
|
||||
key_cache = key_cache.view(-1, self.num_kv_heads, self.head_size)
|
||||
value_cache = value_cache.view(-1, self.num_kv_heads, self.head_size)
|
||||
query, key_tensor, value_tensor = map(
|
||||
|
||||
@@ -768,7 +768,8 @@ class AiterFlashAttentionBackend(AttentionBackend):
|
||||
) -> tuple[int, ...]:
|
||||
if block_size % 16 != 0:
|
||||
raise ValueError("Block size must be a multiple of 16.")
|
||||
return (num_blocks, 2, block_size, num_kv_heads, head_size)
|
||||
# K and V are packed into the content dim: logical (B, H, N, 2*hs).
|
||||
return (num_blocks, num_kv_heads, block_size, 2 * head_size)
|
||||
|
||||
@classmethod
|
||||
def supports_compute_capability(cls, capability: DeviceCapability) -> bool:
|
||||
@@ -1061,7 +1062,8 @@ class AiterFlashAttentionImpl(AttentionImpl):
|
||||
# Whenever making a change in this method, please benchmark the
|
||||
# performance to make sure it does not introduce any overhead.
|
||||
num_actual_tokens = attn_metadata.num_actual_tokens
|
||||
key_cache, value_cache = kv_cache.unbind(1)
|
||||
# (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs))
|
||||
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
|
||||
|
||||
if is_quantized_kv_cache(self.kv_cache_dtype):
|
||||
key_cache = key_cache.view(current_platform.fp8_dtype())
|
||||
@@ -1388,7 +1390,8 @@ class AiterFlashAttentionImpl(AttentionImpl):
|
||||
kv_cache: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
):
|
||||
key_cache, value_cache = kv_cache.unbind(1)
|
||||
# (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs))
|
||||
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
|
||||
|
||||
# key and value may be None in the case of cross attention. They are
|
||||
# calculated once based on the output from the encoder and then cached
|
||||
@@ -1454,7 +1457,8 @@ class AiterFlashAttentionImpl(AttentionImpl):
|
||||
kv_cache: torch.Tensor,
|
||||
layer_slot_mapping: torch.Tensor,
|
||||
):
|
||||
key_cache, value_cache = kv_cache.unbind(1)
|
||||
# (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs))
|
||||
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
|
||||
flash_layout = True
|
||||
|
||||
is_fp8_kv_cache = is_quantized_kv_cache(self.kv_cache_dtype)
|
||||
|
||||
@@ -87,7 +87,8 @@ class RocmAiterUnifiedAttentionBackend(RocmAttentionBackend):
|
||||
) -> tuple[int, ...]:
|
||||
if block_size % 16 != 0:
|
||||
raise ValueError("Block size must be a multiple of 16.")
|
||||
return (num_blocks, 2, block_size, num_kv_heads, head_size)
|
||||
# K and V are packed into the content dim: logical (B, H, N, 2*hs).
|
||||
return (num_blocks, num_kv_heads, block_size, 2 * head_size)
|
||||
|
||||
@staticmethod
|
||||
def use_cascade_attention(*args, **kwargs) -> bool:
|
||||
@@ -150,10 +151,8 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl):
|
||||
def _split_kv_cache(
|
||||
self, kv_cache: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# Blocks-first ``(num_blocks, 2, ...)``. The model runner normalizes any
|
||||
# shared decoder/cross-attention allocation to this layout, so no
|
||||
# per-backend restriding is needed here.
|
||||
return kv_cache.unbind(1)
|
||||
# (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs))
|
||||
return kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
|
||||
@@ -324,6 +324,7 @@ class TritonAttentionBackend(AttentionBackend):
|
||||
) -> tuple[int, ...]:
|
||||
if block_size % 16 != 0:
|
||||
raise ValueError("Block size must be a multiple of 16.")
|
||||
# K and V are packed into the content dim: logical (B, H, N, 2*hs).
|
||||
if kv_cache_uses_per_token_head_scales(cache_dtype_str):
|
||||
# Pad the head dim by sizeof(float32)/sizeof(cache_dtype) so the
|
||||
# per-(token, head) scale fits inline after the quantized data;
|
||||
@@ -341,26 +342,30 @@ class TritonAttentionBackend(AttentionBackend):
|
||||
data_head_size = head_size // 2
|
||||
else:
|
||||
data_head_size = head_size
|
||||
return (num_blocks, 2, block_size, num_kv_heads, data_head_size + scale_pad)
|
||||
return (num_blocks, 2, block_size, num_kv_heads, head_size)
|
||||
padded_hs = data_head_size + scale_pad
|
||||
return (num_blocks, num_kv_heads, block_size, 2 * padded_hs)
|
||||
return (num_blocks, num_kv_heads, block_size, 2 * head_size)
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_stride_order(
|
||||
include_num_layers_dimension: bool = False,
|
||||
) -> tuple[int, ...]:
|
||||
# `stride_order` indicates the permutation that gets
|
||||
# us from `get_kv_cache_shape` to the actual memory layout we want.
|
||||
# `stride_order` indicates the permutation that gets us from
|
||||
# `get_kv_cache_shape` (logical (B, H, N, 2*hs)) to the actual memory
|
||||
# layout we want.
|
||||
cache_layout = get_kv_cache_layout()
|
||||
if cache_layout == "NHD" and include_num_layers_dimension:
|
||||
# (num_blocks, num_layers, 2, block_size, num_kv_heads, head_size)
|
||||
return (1, 0, 2, 3, 4, 5)
|
||||
# (num_blocks, num_layers, block_size, num_kv_heads, 2*head_size)
|
||||
return (1, 0, 3, 2, 4)
|
||||
elif cache_layout == "NHD":
|
||||
stride_order = (0, 1, 2, 3, 4)
|
||||
# (num_blocks, block_size, num_kv_heads, 2*head_size)
|
||||
stride_order = (0, 2, 1, 3)
|
||||
elif cache_layout == "HND" and include_num_layers_dimension:
|
||||
# (num_blocks, num_kv_heads, num_layers, 2, block_size, head_size)
|
||||
return (1, 4, 0, 2, 3, 5)
|
||||
# (num_blocks, num_kv_heads, num_layers, block_size, 2*head_size)
|
||||
return (1, 2, 0, 3, 4)
|
||||
elif cache_layout == "HND":
|
||||
stride_order = (0, 1, 3, 2, 4)
|
||||
# (num_blocks, num_kv_heads, block_size, 2*head_size)
|
||||
stride_order = (0, 1, 2, 3)
|
||||
else:
|
||||
raise ValueError(f"Unknown cache layout: {cache_layout}")
|
||||
return stride_order
|
||||
@@ -410,12 +415,16 @@ class TritonAttentionImpl(AttentionImpl):
|
||||
_v_scale_cache: torch.Tensor | None = None
|
||||
|
||||
def _ensure_scale_caches(self, kv_cache: torch.Tensor) -> None:
|
||||
"""Extract per-head scale views from the padded head dimension.
|
||||
"""Extract per-head scale views from the padded content dimension.
|
||||
|
||||
The KV cache shape is ``(num_blocks, 2, block_size, nkv, hs+pad)``
|
||||
where ``pad = sizeof(float32) / sizeof(cache_dtype)``. The last
|
||||
``pad`` elements of each head hold one float32 scale. We create
|
||||
strided float32 views over those bytes.
|
||||
The KV cache is packed as logical shape
|
||||
``(num_blocks, nkv, block_size, 2 * (hs + pad))`` where
|
||||
``pad = sizeof(float32) / sizeof(cache_dtype)``. The content dim holds
|
||||
``[K(hs) | K_scale(pad) | V(hs) | V_scale(pad)]`` per (head, slot); the
|
||||
last ``pad`` elements of each half hold one float32 scale. We create
|
||||
strided float32 views over those bytes. ``kv_cache`` must be the
|
||||
packed logical tensor (call before any transpose), but may have HND or
|
||||
NHD physical strides.
|
||||
|
||||
Scale shape: ``(num_blocks, block_size, num_kv_heads)``
|
||||
"""
|
||||
@@ -423,9 +432,10 @@ class TritonAttentionImpl(AttentionImpl):
|
||||
return
|
||||
from vllm.utils.torch_utils import get_dtype_size
|
||||
|
||||
num_blocks, _, block_size, nkv, padded_hs = kv_cache.shape
|
||||
num_blocks, nkv, block_size, content = kv_cache.shape
|
||||
dtype_sz = kv_cache.element_size()
|
||||
scale_pad = get_dtype_size(torch.float32) // dtype_sz # e.g. 4
|
||||
padded_hs = content // 2
|
||||
hs = padded_hs - scale_pad
|
||||
|
||||
raw = kv_cache.untyped_storage()
|
||||
@@ -433,31 +443,37 @@ class TritonAttentionImpl(AttentionImpl):
|
||||
raw
|
||||
)
|
||||
|
||||
# In the raw bytes, each (block, kv_half, slot, head) occupies
|
||||
# padded_hs * dtype_sz bytes. The scale float32 sits at byte
|
||||
# offset hs * dtype_sz within that region.
|
||||
kv_half_bytes = block_size * nkv * padded_hs * dtype_sz
|
||||
full_block_f32 = 2 * kv_half_bytes // 4 # stride between blocks
|
||||
slot_f32 = nkv * padded_hs * dtype_sz // 4 # stride between slots
|
||||
head_f32 = padded_hs * dtype_sz // 4 # stride between heads
|
||||
scale_off_f32 = hs * dtype_sz // 4 # offset to scale within head
|
||||
def to_f32_units(elements: int) -> int:
|
||||
nbytes = elements * dtype_sz
|
||||
assert nbytes % 4 == 0
|
||||
return nbytes // 4
|
||||
|
||||
# K scales: kv_half=0
|
||||
# Actual strides (in float32 units) from the tensor. The logical cache
|
||||
# may be physically NHD, so do not assume C-contiguous HND layout.
|
||||
strides = kv_cache.stride()
|
||||
block_f32 = to_f32_units(strides[0])
|
||||
head_f32 = to_f32_units(strides[1])
|
||||
slot_f32 = to_f32_units(strides[2])
|
||||
# Scale sits at byte offset hs within each (K, then V) content half.
|
||||
base_off_f32 = to_f32_units(kv_cache.storage_offset())
|
||||
k_scale_off_f32 = base_off_f32 + to_f32_units(hs)
|
||||
v_scale_off_f32 = base_off_f32 + to_f32_units(padded_hs + hs)
|
||||
|
||||
# K scales (first content half)
|
||||
self._k_scale_cache = torch.as_strided(
|
||||
base_f32,
|
||||
size=(num_blocks, block_size, nkv),
|
||||
stride=(full_block_f32, slot_f32, head_f32),
|
||||
storage_offset=scale_off_f32,
|
||||
stride=(block_f32, slot_f32, head_f32),
|
||||
storage_offset=k_scale_off_f32,
|
||||
)
|
||||
self._k_scale_cache.fill_(1.0)
|
||||
|
||||
# V scales: kv_half=1, offset by kv_half_bytes
|
||||
v_base_f32 = kv_half_bytes // 4
|
||||
# V scales (second content half)
|
||||
self._v_scale_cache = torch.as_strided(
|
||||
base_f32,
|
||||
size=(num_blocks, block_size, nkv),
|
||||
stride=(full_block_f32, slot_f32, head_f32),
|
||||
storage_offset=v_base_f32 + scale_off_f32,
|
||||
stride=(block_f32, slot_f32, head_f32),
|
||||
storage_offset=v_scale_off_f32,
|
||||
)
|
||||
self._v_scale_cache.fill_(1.0)
|
||||
|
||||
@@ -576,7 +592,7 @@ class TritonAttentionImpl(AttentionImpl):
|
||||
key: shape = [num_tokens, num_kv_heads, head_size]
|
||||
value: shape = [num_tokens, num_kv_heads, head_size]
|
||||
kv_cache: shape =
|
||||
[num_blocks, 2, block_size, num_kv_heads, head_size]
|
||||
[num_blocks, num_kv_heads, block_size, 2 * head_size]
|
||||
attn_metadata: Metadata for attention.
|
||||
Returns:
|
||||
shape = [num_tokens, num_heads * head_size]
|
||||
@@ -617,6 +633,7 @@ class TritonAttentionImpl(AttentionImpl):
|
||||
layer,
|
||||
)
|
||||
|
||||
# KV cache arrives in logical (B, H, N, 2*hs) order.
|
||||
# Per-token-head quantized KV cache: handled by the core unified
|
||||
# kernel, which dequantizes per-(token, head) inline via constexpr
|
||||
# branches (INT8 / FP8) and dispatches to the packed INT4 kernel.
|
||||
@@ -627,7 +644,9 @@ class TritonAttentionImpl(AttentionImpl):
|
||||
q_descale = k_descale = v_descale = None
|
||||
# FP8 per-tensor / auto path (original flow).
|
||||
else:
|
||||
key_cache, value_cache = kv_cache.unbind(1)
|
||||
kv_cache = kv_cache.transpose(1, 2)
|
||||
hs = self.head_size
|
||||
key_cache, value_cache = kv_cache.split(hs, dim=-1)
|
||||
if (
|
||||
is_quantized_kv_cache(self.kv_cache_dtype)
|
||||
and key_cache.dtype != self.fp8_dtype
|
||||
@@ -711,7 +730,8 @@ class TritonAttentionImpl(AttentionImpl):
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Per-token-head K/V cache views (ensures scale caches; FP8 retyped)."""
|
||||
self._ensure_scale_caches(kv_cache)
|
||||
key_cache, value_cache = kv_cache.unbind(1)
|
||||
padded_hs = kv_cache.shape[-1] // 2
|
||||
key_cache, value_cache = kv_cache.transpose(1, 2).split(padded_hs, dim=-1)
|
||||
if self._kv_quant_mode == KVQuantMode.FP8_PER_TOKEN_HEAD:
|
||||
key_cache = key_cache.view(self.fp8_dtype)
|
||||
value_cache = value_cache.view(self.fp8_dtype)
|
||||
@@ -778,13 +798,9 @@ class TritonAttentionImpl(AttentionImpl):
|
||||
return
|
||||
# Reshape the input keys and values and store them in the cache.
|
||||
if self._is_per_token_head_quant:
|
||||
self._ensure_scale_caches(kv_cache)
|
||||
key_cache, value_cache = kv_cache.unbind(1)
|
||||
key_cache, value_cache = self._pth_key_value_caches(kv_cache)
|
||||
k_scale_cache = self._k_scale_cache
|
||||
v_scale_cache = self._v_scale_cache
|
||||
if self._kv_quant_mode == KVQuantMode.FP8_PER_TOKEN_HEAD:
|
||||
key_cache = key_cache.view(self.fp8_dtype)
|
||||
value_cache = value_cache.view(self.fp8_dtype)
|
||||
triton_reshape_and_cache_flash_per_token_head_quant(
|
||||
key,
|
||||
value,
|
||||
@@ -797,7 +813,8 @@ class TritonAttentionImpl(AttentionImpl):
|
||||
)
|
||||
return
|
||||
# For decoder and cross-attention, use KV cache as before.
|
||||
key_cache, value_cache = kv_cache.unbind(1)
|
||||
# (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs))
|
||||
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
|
||||
if is_quantized_kv_cache(self.kv_cache_dtype):
|
||||
key_cache = key_cache.view(self.fp8_dtype)
|
||||
value_cache = value_cache.view(self.fp8_dtype)
|
||||
@@ -829,7 +846,8 @@ class TritonAttentionImpl(AttentionImpl):
|
||||
kv_cache: torch.Tensor,
|
||||
layer_slot_mapping: torch.Tensor,
|
||||
):
|
||||
key_cache, value_cache = kv_cache.unbind(1)
|
||||
# (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs))
|
||||
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
|
||||
flash_layout = True
|
||||
|
||||
is_fp8_kv_cache = is_quantized_kv_cache(self.kv_cache_dtype)
|
||||
|
||||
@@ -2,12 +2,9 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Triton attention backend with different K/V head dimensions (DiffKV).
|
||||
|
||||
The KV cache layout is identical to ``FlashAttentionDiffKVBackend`` — K
|
||||
and V are packed along the last dim:
|
||||
|
||||
[num_blocks, block_size, num_kv_heads, head_size_qk + head_size_v]
|
||||
|
||||
so existing helpers (``triton_reshape_and_cache_flash_diffkv``) are reused.
|
||||
The KV cache layout is identical to ``FlashAttentionDiffKVBackend``: K and V
|
||||
are packed along the last dim in the logical shape
|
||||
``[num_blocks, num_kv_heads, block_size, head_size_qk + head_size_v]``.
|
||||
"""
|
||||
|
||||
from typing import ClassVar
|
||||
@@ -106,10 +103,12 @@ class TritonAttentionDiffKVBackend(TritonAttentionBackend):
|
||||
) -> tuple[int, ...]:
|
||||
if block_size % 16 != 0:
|
||||
raise ValueError("Block size must be a multiple of 16.")
|
||||
# Logical (blocks-first, head-major) layout: K and V (with their
|
||||
# different head sizes) packed in the content dim.
|
||||
return (
|
||||
num_blocks,
|
||||
block_size,
|
||||
num_kv_heads,
|
||||
block_size,
|
||||
head_size + TritonAttentionDiffKVBackend.head_size_v,
|
||||
)
|
||||
|
||||
@@ -117,19 +116,22 @@ class TritonAttentionDiffKVBackend(TritonAttentionBackend):
|
||||
def get_kv_cache_stride_order(
|
||||
include_num_layers_dimension: bool = False,
|
||||
) -> tuple[int, ...]:
|
||||
# `stride_order` indicates the permutation that gets us from
|
||||
# `get_kv_cache_shape` (logical (B, H, N, C_k+C_v)) to the actual
|
||||
# memory layout we want.
|
||||
cache_layout = get_kv_cache_layout()
|
||||
if cache_layout == "NHD" and include_num_layers_dimension:
|
||||
# (num_blocks, num_layers, block_size,
|
||||
# num_kv_heads, head_size + head_size_v)
|
||||
return (1, 0, 2, 3, 4)
|
||||
# (num_blocks, num_layers, block_size, num_kv_heads, C_k+C_v)
|
||||
return (1, 0, 3, 2, 4)
|
||||
elif cache_layout == "NHD":
|
||||
return (0, 1, 2, 3)
|
||||
elif cache_layout == "HND" and include_num_layers_dimension:
|
||||
# (num_blocks, num_kv_heads, num_layers,
|
||||
# block_size, head_size + head_size_v)
|
||||
return (1, 3, 0, 2, 4)
|
||||
elif cache_layout == "HND":
|
||||
# (num_blocks, block_size, num_kv_heads, C_k+C_v)
|
||||
return (0, 2, 1, 3)
|
||||
elif cache_layout == "HND" and include_num_layers_dimension:
|
||||
# (num_blocks, num_kv_heads, num_layers, block_size, C_k+C_v)
|
||||
return (1, 2, 0, 3, 4)
|
||||
elif cache_layout == "HND":
|
||||
# (num_blocks, num_kv_heads, block_size, C_k+C_v)
|
||||
return (0, 1, 2, 3)
|
||||
else:
|
||||
raise ValueError(f"Unknown cache layout format {cache_layout}.")
|
||||
|
||||
@@ -175,13 +177,12 @@ class TritonAttentionDiffKVImpl(TritonAttentionImpl):
|
||||
kv_cache: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
) -> None:
|
||||
# Cache is packed [..., head_size_qk + head_size_v]; the diffkv
|
||||
# reshape kernel writes K to [..., :head_size_qk] and V to
|
||||
# [..., head_size_qk:hqk+hv].
|
||||
# Cache is logical (B, H, N, C); the diffkv reshape kernel expects
|
||||
# (B, N, H, C).
|
||||
triton_reshape_and_cache_flash_diffkv(
|
||||
key,
|
||||
value,
|
||||
kv_cache,
|
||||
kv_cache.transpose(1, 2),
|
||||
slot_mapping,
|
||||
self.kv_cache_dtype,
|
||||
layer._k_scale,
|
||||
@@ -210,7 +211,7 @@ class TritonAttentionDiffKVImpl(TritonAttentionImpl):
|
||||
query: [num_tokens, num_heads, head_size_qk]
|
||||
key: [num_tokens, num_kv_heads, head_size_qk]
|
||||
value: [num_tokens, num_kv_heads, head_size_v]
|
||||
kv_cache: [num_blocks, block_size, num_kv_heads,
|
||||
kv_cache: [num_blocks, num_kv_heads, block_size,
|
||||
head_size_qk + head_size_v]
|
||||
output: [num_tokens, num_heads, head_size_v]
|
||||
"""
|
||||
@@ -231,8 +232,8 @@ class TritonAttentionDiffKVImpl(TritonAttentionImpl):
|
||||
head_size_qk = self.head_size
|
||||
head_size_v = TritonAttentionDiffKVBackend.head_size_v
|
||||
|
||||
# Slice the packed cache into K / V views. Strides on dims 0/1/2
|
||||
# match the original cache; dim 3 stays contiguous (stride 1).
|
||||
# Triton DiffKV kernels consume (B, N, H, D) cache views.
|
||||
kv_cache = kv_cache.transpose(1, 2)
|
||||
key_cache = kv_cache[..., :head_size_qk]
|
||||
value_cache = kv_cache[..., head_size_qk : head_size_qk + head_size_v]
|
||||
|
||||
|
||||
@@ -141,9 +141,10 @@ class TurboQuantAttentionBackend(AttentionBackend):
|
||||
|
||||
Standard attention backends use (2, num_blocks, block_size, num_kv_heads,
|
||||
head_dim) with a leading 2 to separate K and V. TurboQuant packs K+V
|
||||
into a single interleaved slot per head per position, so the cache is:
|
||||
into a single interleaved slot per head per position. The logical
|
||||
(blocks-first, head-major) shape is:
|
||||
|
||||
(num_blocks, block_size, num_kv_heads, slot_size_aligned)
|
||||
(num_blocks, num_kv_heads, block_size, slot_size_aligned)
|
||||
|
||||
Each slot = [key_packed | value_packed | padding].
|
||||
This is safe because TQ has its own get_kv_cache_shape override and
|
||||
@@ -159,7 +160,7 @@ class TurboQuantAttentionBackend(AttentionBackend):
|
||||
)
|
||||
|
||||
tq_config = TurboQuantConfig.from_cache_dtype(cache_dtype_str, head_size)
|
||||
return (num_blocks, block_size, num_kv_heads, tq_config.slot_size_aligned)
|
||||
return (num_blocks, num_kv_heads, block_size, tq_config.slot_size_aligned)
|
||||
|
||||
@classmethod
|
||||
def supports_kv_cache_dtype(cls, kv_cache_dtype: CacheDType | None) -> bool:
|
||||
@@ -422,6 +423,8 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]):
|
||||
|
||||
k = key[:N].view(N, self.num_kv_heads, self.head_size)
|
||||
v = value[:N].view(N, self.num_kv_heads, self.head_size)
|
||||
# (B, H, N, C) -> (B, N, H, C) for TQ kernels
|
||||
kv_cache = kv_cache.transpose(1, 2)
|
||||
self._store_kv(k, v, kv_cache, slot_mapping, layer)
|
||||
|
||||
def forward(
|
||||
@@ -449,6 +452,9 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]):
|
||||
if attn_metadata is None:
|
||||
return output.fill_(0)
|
||||
|
||||
# (B, H, N, C) -> (B, N, H, C) for TQ kernels
|
||||
kv_cache = kv_cache.transpose(1, 2)
|
||||
|
||||
# Slice to actual tokens
|
||||
N = attn_metadata.num_actual_tokens
|
||||
if N <= 0:
|
||||
|
||||
@@ -473,6 +473,7 @@ def reshape_and_cache_kernel_flash_diffkv(
|
||||
value_stride: tl.int64,
|
||||
block_stride: tl.int64,
|
||||
page_stride: tl.int64,
|
||||
head_stride: tl.int64,
|
||||
num_heads: tl.constexpr,
|
||||
head_size_k: tl.constexpr,
|
||||
head_size_v: tl.constexpr,
|
||||
@@ -498,9 +499,7 @@ def reshape_and_cache_kernel_flash_diffkv(
|
||||
src_value_idx = token_idx * value_stride + tile_i * head_size_v
|
||||
|
||||
tgt_idx = (
|
||||
block_idx * block_stride
|
||||
+ block_offset * page_stride
|
||||
+ tile_i * (head_size_k + head_size_v)
|
||||
block_idx * block_stride + block_offset * page_stride + tile_i * head_stride
|
||||
)
|
||||
|
||||
# [TILE_SIZE]
|
||||
@@ -542,7 +541,7 @@ def reshape_and_cache_kernel_flash_diffkv(
|
||||
def triton_reshape_and_cache_flash_diffkv(
|
||||
key: torch.Tensor, # [num_tokens, num_heads, head_size]
|
||||
value: torch.Tensor, # [num_tokens, num_heads, head_size_v]
|
||||
# [num_blocks, block_size, num_heads, head_size + head_size_v]
|
||||
# Strided [num_blocks, block_size, num_heads, head_size + head_size_v].
|
||||
kv_cache: torch.Tensor,
|
||||
slot_mapping: torch.Tensor, # [num_tokens]
|
||||
kv_cache_dtype: str, # "auto", "fp8"
|
||||
@@ -558,6 +557,7 @@ def triton_reshape_and_cache_flash_diffkv(
|
||||
v_stride = value.stride()[0]
|
||||
block_stride = kv_cache.stride()[0]
|
||||
page_stride = kv_cache.stride()[1]
|
||||
head_stride = kv_cache.stride()[2]
|
||||
|
||||
kv_cache_torch_dtype = (
|
||||
current_platform.fp8_dtype()
|
||||
@@ -599,6 +599,7 @@ def triton_reshape_and_cache_flash_diffkv(
|
||||
value_stride=v_stride,
|
||||
block_stride=block_stride,
|
||||
page_stride=page_stride,
|
||||
head_stride=head_stride,
|
||||
num_heads=num_heads,
|
||||
head_size_k=head_size_k,
|
||||
head_size_v=head_size_v,
|
||||
|
||||
@@ -389,7 +389,7 @@ def triton_turboquant_store(
|
||||
_tq_fused_store_fp8[grid](
|
||||
k_flat,
|
||||
v_flat,
|
||||
kv_cache.view(-1),
|
||||
kv_cache,
|
||||
slot_mapping,
|
||||
stride_cache_block=stride_block,
|
||||
stride_cache_pos=stride_pos,
|
||||
@@ -425,7 +425,7 @@ def triton_turboquant_store(
|
||||
norms.squeeze(1),
|
||||
v_flat,
|
||||
midpoints,
|
||||
kv_cache.view(-1),
|
||||
kv_cache,
|
||||
slot_mapping,
|
||||
stride_cache_block=stride_block,
|
||||
stride_cache_pos=stride_pos,
|
||||
|
||||
Reference in New Issue
Block a user