[6/N][KV-Cache Layout Refactor] Standardize KV cache layout (#51718)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
@@ -356,21 +356,21 @@ def test_reshape_and_cache_flash(
|
||||
dequant_nvfp4_kv_cache,
|
||||
)
|
||||
|
||||
def dequant_nvfp4_cache_nhd(data_cache, scale_cache, global_scale):
|
||||
# data_cache: [N, T, H, data_dim] NHD (contiguous inner dims)
|
||||
# scale_cache: [N, T, H, scale_dim] NHD (contiguous inner dims)
|
||||
# Permute to HND layout for the dequant utility.
|
||||
def dequant_nvfp4_cache(data_cache, scale_cache, global_scale):
|
||||
# data_cache: [B, N, H, data_dim] logical view (layout in strides)
|
||||
# scale_cache: [B, N, H, scale_dim] logical view (layout in strides)
|
||||
# Permute to [B, H, N, dim] for the dequant utility.
|
||||
data_hnd = data_cache.permute(0, 2, 1, 3)
|
||||
scale_hnd = scale_cache.permute(0, 2, 1, 3)
|
||||
result_hnd = dequant_nvfp4_kv_cache(
|
||||
data_hnd, scale_hnd, global_scale, head_size, block_size
|
||||
)
|
||||
return result_hnd.permute(0, 2, 1, 3) # back to [N, T, H, D]
|
||||
return result_hnd.permute(0, 2, 1, 3) # back to [B, N, H, dim]
|
||||
|
||||
result_key_cache = dequant_nvfp4_cache_nhd(
|
||||
result_key_cache = dequant_nvfp4_cache(
|
||||
nvfp4_key_data, key_scale_cache, k_scale.item()
|
||||
)
|
||||
result_value_cache = dequant_nvfp4_cache_nhd(
|
||||
result_value_cache = dequant_nvfp4_cache(
|
||||
nvfp4_value_data, value_scale_cache, v_scale.item()
|
||||
)
|
||||
|
||||
|
||||
@@ -5,10 +5,10 @@
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.v1.attention.utils import dense_kv_cache_views
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.models.minimax_m3.common.indexer import (
|
||||
MiniMaxM3IndexerBackend,
|
||||
)
|
||||
from vllm.config import CacheConfig
|
||||
from vllm.config.cache import _layout_from_name
|
||||
from vllm.models.minimax_m3.common.ops.index_topk import (
|
||||
minimax_m3_index_decode,
|
||||
minimax_m3_index_score,
|
||||
@@ -19,15 +19,15 @@ from vllm.models.minimax_m3.common.ops.sparse_attn import (
|
||||
minimax_m3_sparse_attn,
|
||||
minimax_m3_sparse_attn_decode,
|
||||
)
|
||||
from vllm.models.minimax_m3.common.sparse_attention import (
|
||||
MiniMaxM3SparseBackend,
|
||||
MiniMaxM3SparseTritonImpl,
|
||||
)
|
||||
from vllm.models.minimax_m3.common.sparse_attention import MiniMaxM3SparseTritonImpl
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.utils import set_kv_cache_layout
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, MLAAttentionSpec
|
||||
from vllm.v1.worker.gpu.attn_utils import _reshape_kv_cache
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
from vllm.v1.attention.backends.utils import record_kv_cache_layout
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheLayout,
|
||||
MLAAttentionSpec,
|
||||
compute_layer_kv_cache_shape_bytes,
|
||||
)
|
||||
|
||||
if not (current_platform.is_cuda() or current_platform.is_rocm()):
|
||||
pytest.skip(
|
||||
@@ -37,35 +37,46 @@ if not (current_platform.is_cuda() or current_platform.is_rocm()):
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def kv_layout(request):
|
||||
"""Set the global KV cache layout for one test and restore it after."""
|
||||
set_kv_cache_layout(request.param)
|
||||
try:
|
||||
yield request.param
|
||||
finally:
|
||||
set_kv_cache_layout(None)
|
||||
def kv_layout(request) -> KVCacheLayout:
|
||||
"""Resolve the requested layout name (legacy NHD/HND aliases included)."""
|
||||
return _layout_from_name(request.param)
|
||||
|
||||
|
||||
def _stride_order_for(backend: type[MiniMaxM3SparseBackend], ndim: int) -> tuple:
|
||||
"""Mirror the allocator's stride-order resolution (identity fallback)."""
|
||||
try:
|
||||
stride_order = backend.get_kv_cache_stride_order()
|
||||
assert len(stride_order) == ndim
|
||||
except (AttributeError, NotImplementedError):
|
||||
stride_order = tuple(range(ndim))
|
||||
def _layer_stride_order(layout: KVCacheLayout, ndim: int) -> tuple[int, ...]:
|
||||
"""Per-layer physical stride order for the given layout; the 3-dim
|
||||
indexer side cache (H=1) is contiguous, so identity."""
|
||||
if ndim == 3:
|
||||
return (0, 1, 2)
|
||||
stride_order = layout.layer_view_order
|
||||
assert len(stride_order) == ndim
|
||||
return stride_order
|
||||
|
||||
|
||||
def _main_spec() -> FullAttentionSpec:
|
||||
return FullAttentionSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=NUM_KV_HEADS,
|
||||
head_size=HEAD_DIM,
|
||||
head_size_v=HEAD_DIM,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
|
||||
|
||||
def _main_kv_logical_shape(num_pages: int) -> tuple[int, ...]:
|
||||
"""Standardized per-layer logical shape [B, H, N, C] for the main cache,
|
||||
derived the same way the production allocator does."""
|
||||
shape_bytes = compute_layer_kv_cache_shape_bytes(_main_spec(), num_pages)
|
||||
return (*shape_bytes[:-1], shape_bytes[-1] // DTYPE.itemsize)
|
||||
|
||||
|
||||
def _allocate_main_kv_via_contract(
|
||||
num_pages: int, device: torch.device | str = "cuda"
|
||||
num_pages: int, layout: KVCacheLayout, device: torch.device | str = "cuda"
|
||||
) -> torch.Tensor:
|
||||
"""Build the main KV cache exactly as the production allocator does for the
|
||||
currently active layout: allocate the physical (permuted) tensor, then
|
||||
expose the inverse-permuted logical-NHD view the backend sees."""
|
||||
logical_shape = MiniMaxM3SparseBackend.get_kv_cache_shape(
|
||||
num_pages, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
|
||||
)
|
||||
stride_order = _stride_order_for(MiniMaxM3SparseBackend, len(logical_shape))
|
||||
given layout: allocate the physical (permuted) tensor, then expose the
|
||||
inverse-permuted logical [B, H, N, C] view the backend sees."""
|
||||
logical_shape = _main_kv_logical_shape(num_pages)
|
||||
stride_order = _layer_stride_order(layout, len(logical_shape))
|
||||
physical_shape = tuple(logical_shape[i] for i in stride_order)
|
||||
inv_order = [stride_order.index(i) for i in range(len(stride_order))]
|
||||
raw = torch.randn(physical_shape, device=device, dtype=DTYPE)
|
||||
@@ -817,7 +828,7 @@ def _reference_sparse_attn(
|
||||
],
|
||||
)
|
||||
def test_prefill_sparse_attention_correctness(
|
||||
kv_layout: str,
|
||||
kv_layout: KVCacheLayout,
|
||||
q_lens: tuple[int, ...],
|
||||
kv_lens: tuple[int, ...],
|
||||
):
|
||||
@@ -851,7 +862,7 @@ def test_prefill_sparse_attention_correctness(
|
||||
# Allocate the main KV cache through the backend layout contract so the
|
||||
# physical storage matches the active layout (contiguous NHD or strided
|
||||
# HND), while the kernels and reference see the logical-NHD view.
|
||||
kv_cache = _allocate_main_kv_via_contract(num_pages)
|
||||
kv_cache = _allocate_main_kv_via_contract(num_pages, kv_layout)
|
||||
|
||||
# Build sparse block indices with the same contract as the real M3 indexer:
|
||||
# one forced local block, then score-selected older causal blocks.
|
||||
@@ -904,38 +915,31 @@ def test_prefill_sparse_attention_correctness(
|
||||
assert error.max().item() < 1.7e-2
|
||||
|
||||
|
||||
def test_main_backend_layout_contract():
|
||||
"""The main sparse backend exposes the logical-NHD shape and the
|
||||
flash_attn-style stride order for each layout."""
|
||||
def test_main_cache_layout_contract():
|
||||
"""The standardized per-layer logical shape is [B, H, N, C] with packed
|
||||
K/V content, and the legacy layout aliases resolve to the expected
|
||||
per-layer stride orders."""
|
||||
nb, bs, h, d = 7, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
|
||||
logical = MiniMaxM3SparseBackend.get_kv_cache_shape(nb, bs, h, d)
|
||||
logical = _main_kv_logical_shape(nb)
|
||||
assert logical == (nb, h, bs, 2 * d)
|
||||
# The old separate K/V-axis shape is no longer the logical shape.
|
||||
assert logical != (nb, 2, bs, h, d)
|
||||
|
||||
try:
|
||||
set_kv_cache_layout("HND")
|
||||
assert MiniMaxM3SparseBackend.get_kv_cache_stride_order() == (0, 1, 2, 3)
|
||||
set_kv_cache_layout("NHD")
|
||||
assert MiniMaxM3SparseBackend.get_kv_cache_stride_order() == (0, 2, 1, 3)
|
||||
finally:
|
||||
set_kv_cache_layout(None)
|
||||
assert KVCacheLayout.LBHNC.layer_view_order == (0, 1, 2, 3)
|
||||
assert KVCacheLayout.LBNHC.layer_view_order == (0, 2, 1, 3)
|
||||
|
||||
for layout in ("NHD", "HND"):
|
||||
try:
|
||||
set_kv_cache_layout(layout)
|
||||
order = MiniMaxM3SparseBackend.get_kv_cache_stride_order()
|
||||
finally:
|
||||
set_kv_cache_layout(None)
|
||||
cache_config = CacheConfig()
|
||||
record_kv_cache_layout(cache_config, "HND")
|
||||
assert cache_config.get_resolved_kv_cache_layout() is KVCacheLayout.LBHNC
|
||||
cache_config = CacheConfig()
|
||||
record_kv_cache_layout(cache_config, "NHD")
|
||||
assert cache_config.get_resolved_kv_cache_layout() is KVCacheLayout.LBNHC
|
||||
|
||||
for layout in (KVCacheLayout.LBNHC, KVCacheLayout.LBHNC):
|
||||
order = layout.layer_view_order
|
||||
# Valid permutation: no duplicates, covers every axis.
|
||||
assert set(order) == set(range(len(order)))
|
||||
|
||||
# M3 has no cross-layer KV blocks.
|
||||
with pytest.raises(NotImplementedError):
|
||||
MiniMaxM3SparseBackend.get_kv_cache_stride_order(
|
||||
include_num_layers_dimension=True
|
||||
)
|
||||
|
||||
|
||||
def test_aiter_sparse_pa_layout_contract(monkeypatch):
|
||||
"""The shuffle-only AITER path retains separately contiguous K/V storage."""
|
||||
@@ -948,75 +952,53 @@ def test_aiter_sparse_pa_layout_contract(monkeypatch):
|
||||
lambda: True,
|
||||
)
|
||||
|
||||
nb, bs, h, d = 7, BLOCK_SIZE, 1, HEAD_DIM
|
||||
logical = MiniMaxM3SparseBackend.get_kv_cache_shape(nb, bs, h, d)
|
||||
order = MiniMaxM3SparseBackend.get_kv_cache_stride_order()
|
||||
assert logical == (nb, 2, bs, h, d)
|
||||
assert order == (1, 0, 2, 3, 4)
|
||||
backend = sparse_attn_mod.MiniMaxM3SparseBackend
|
||||
assert KVCacheLayout.LHBNC in backend.supported_kv_cache_layouts()
|
||||
|
||||
physical_shape = tuple(logical[i] for i in order)
|
||||
inv_order = [order.index(i) for i in range(len(order))]
|
||||
raw = torch.empty(physical_shape, device="cuda", dtype=DTYPE)
|
||||
logical_view = raw.permute(*inv_order)
|
||||
key_cache, value_cache = logical_view.unbind(1)
|
||||
nb, h = 7, 1
|
||||
spec = FullAttentionSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=h,
|
||||
head_size=HEAD_DIM,
|
||||
dtype=DTYPE,
|
||||
num_head_slots=2,
|
||||
state_content_bytes=h * HEAD_DIM * DTYPE.itemsize,
|
||||
)
|
||||
raw = torch.zeros(nb * spec.page_size_bytes, dtype=torch.int8)
|
||||
view = dense_kv_cache_views(raw, spec, nb, 1, KVCacheLayout.LHBNC)[0]
|
||||
key_cache, value_cache = view.unbind(1)
|
||||
assert key_cache.is_contiguous()
|
||||
assert value_cache.is_contiguous()
|
||||
|
||||
|
||||
def test_aiter_sparse_pa_rejects_multiple_kv_heads(monkeypatch):
|
||||
"""Do not pair AITER's separated cache layout with the Triton fallback."""
|
||||
import vllm.models.minimax_m3.common.sparse_attention as sparse_attn_mod
|
||||
|
||||
monkeypatch.setattr(sparse_attn_mod.rocm_aiter_ops, "is_enabled", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
sparse_attn_mod.rocm_aiter_ops,
|
||||
"is_shuffle_kv_cache_enabled",
|
||||
lambda: True,
|
||||
def test_indexer_cache_squeezes_to_contiguous_3d():
|
||||
"""The indexer side cache is standardized 4D with H=1: under both layouts
|
||||
the allocator's logical view stays contiguous and squeezes (as
|
||||
`MiniMaxM3IndexerCache.bind_kv_cache` does) to the 3-dim
|
||||
[num_blocks, block_size, head_dim] cache the kernels consume."""
|
||||
nb = 5
|
||||
ispec = MLAAttentionSpec(
|
||||
block_size=BLOCK_SIZE, num_kv_heads=1, head_size=HEAD_DIM, dtype=DTYPE
|
||||
)
|
||||
shape_bytes = compute_layer_kv_cache_shape_bytes(ispec, nb)
|
||||
assert shape_bytes == (nb, 1, BLOCK_SIZE, HEAD_DIM * DTYPE.itemsize)
|
||||
assert _layer_stride_order(KVCacheLayout.LBNHC, 3) == (0, 1, 2)
|
||||
|
||||
with pytest.raises(ValueError, match="num_kv_heads == 1"):
|
||||
MiniMaxM3SparseBackend.get_kv_cache_shape(7, BLOCK_SIZE, 2, HEAD_DIM)
|
||||
|
||||
|
||||
def test_main_backend_unknown_layout_raises(monkeypatch):
|
||||
"""An unrecognized layout (injected past env-var validation) is rejected."""
|
||||
import vllm.models.minimax_m3.common.sparse_attention as sparse_attn_mod
|
||||
|
||||
monkeypatch.setattr(sparse_attn_mod, "get_kv_cache_layout", lambda: "BOGUS")
|
||||
with pytest.raises(ValueError, match="Unknown cache layout format"):
|
||||
MiniMaxM3SparseBackend.get_kv_cache_stride_order()
|
||||
|
||||
|
||||
def test_indexer_backend_stride_order_is_identity():
|
||||
"""The 3-dim indexer cache must not inherit the parent's 4-element stride
|
||||
order; it overrides to the 3-element identity so the allocator keeps the
|
||||
contiguous layout."""
|
||||
assert MiniMaxM3IndexerBackend.get_kv_cache_stride_order() == (0, 1, 2)
|
||||
|
||||
# Cross-layer (per-layer-stacked) KV blocks are not supported.
|
||||
with pytest.raises(NotImplementedError):
|
||||
MiniMaxM3IndexerBackend.get_kv_cache_stride_order(
|
||||
include_num_layers_dimension=True
|
||||
)
|
||||
|
||||
# The stride order matches the 3-dim indexer shape rank.
|
||||
indexer_shape = MiniMaxM3IndexerBackend.get_kv_cache_shape(
|
||||
5, BLOCK_SIZE, 1, HEAD_DIM
|
||||
)
|
||||
assert len(indexer_shape) == 3
|
||||
assert _stride_order_for(MiniMaxM3IndexerBackend, len(indexer_shape)) == (0, 1, 2)
|
||||
for layout in (KVCacheLayout.LBNHC, KVCacheLayout.LBHNC):
|
||||
iraw = torch.zeros(nb * ispec.page_size_bytes, dtype=torch.int8)
|
||||
view = dense_kv_cache_views(iraw, ispec, nb, 1, layout, BLOCK_SIZE)[0]
|
||||
assert tuple(view.shape) == (nb, 1, BLOCK_SIZE, HEAD_DIM)
|
||||
indexer_cache = view.squeeze(1)
|
||||
assert tuple(indexer_cache.shape) == (nb, BLOCK_SIZE, HEAD_DIM)
|
||||
assert indexer_cache.is_contiguous()
|
||||
|
||||
|
||||
def test_hnd_allocation_is_packed_head_major():
|
||||
"""Under HND the backend-visible logical view is the packed head-major
|
||||
physical allocation."""
|
||||
nb, bs, h, d = 4, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
|
||||
logical = MiniMaxM3SparseBackend.get_kv_cache_shape(nb, bs, h, d)
|
||||
try:
|
||||
set_kv_cache_layout("HND")
|
||||
stride_order = MiniMaxM3SparseBackend.get_kv_cache_stride_order()
|
||||
finally:
|
||||
set_kv_cache_layout(None)
|
||||
logical = _main_kv_logical_shape(nb)
|
||||
stride_order = KVCacheLayout.LBHNC.layer_view_order
|
||||
|
||||
physical_shape = tuple(logical[i] for i in stride_order)
|
||||
assert physical_shape == (nb, h, bs, 2 * d)
|
||||
@@ -1042,27 +1024,13 @@ def test_main_cache_is_block_first_and_unpadded():
|
||||
"""The allocator's contiguous-view branch (not the padded-strided branch)
|
||||
is used for the main GQA cache: its spec is unpadded and the physical
|
||||
layout keeps num_blocks as the first dimension under both layouts."""
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec
|
||||
|
||||
spec = FullAttentionSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=NUM_KV_HEADS,
|
||||
head_size=HEAD_DIM,
|
||||
head_size_v=HEAD_DIM,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
spec = _main_spec()
|
||||
# Unpadded -> allocator uses kv_tensor.view(...) rather than as_strided().
|
||||
assert spec.page_size_padded is None
|
||||
|
||||
logical = MiniMaxM3SparseBackend.get_kv_cache_shape(
|
||||
4, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
|
||||
)
|
||||
for layout in ("NHD", "HND"):
|
||||
try:
|
||||
set_kv_cache_layout(layout)
|
||||
order = MiniMaxM3SparseBackend.get_kv_cache_stride_order()
|
||||
finally:
|
||||
set_kv_cache_layout(None)
|
||||
logical = _main_kv_logical_shape(4)
|
||||
for layout in (KVCacheLayout.LBNHC, KVCacheLayout.LBHNC):
|
||||
order = layout.layer_view_order
|
||||
inv_order = [order.index(i) for i in range(len(order))]
|
||||
# Physical first dim is num_blocks (block-first); required by the
|
||||
# padded-strided branch's block-first assumption if it were ever taken.
|
||||
@@ -1275,7 +1243,7 @@ def test_aiter_decode_sparse_block_table_supports_spec_decode():
|
||||
@pytest.mark.parametrize("decode_query_len", [1, 4])
|
||||
@pytest.mark.parametrize("num_padded_reqs", [0, 2])
|
||||
def test_decode_sparse_attention_correctness(
|
||||
kv_layout: str,
|
||||
kv_layout: KVCacheLayout,
|
||||
seq_lens_list: tuple[int, ...],
|
||||
decode_query_len: int,
|
||||
num_padded_reqs: int,
|
||||
@@ -1287,7 +1255,7 @@ def test_decode_sparse_attention_correctness(
|
||||
q, block_table, seq_lens, topk_idx, num_pages = _build_decode_inputs(
|
||||
seq_lens_list, decode_query_len, num_padded_reqs
|
||||
)
|
||||
kv_cache = _allocate_main_kv_via_contract(num_pages)
|
||||
kv_cache = _allocate_main_kv_via_contract(num_pages, kv_layout)
|
||||
|
||||
actual = torch.empty_like(q)
|
||||
minimax_m3_sparse_attn_decode(
|
||||
@@ -1364,93 +1332,20 @@ def test_decode_wrong_layout_breaks_parity():
|
||||
assert (actual.float() - expected.float()).abs().max().item() > 1.7e-2
|
||||
|
||||
|
||||
def _make_attn_group(backend, spec):
|
||||
return AttentionGroup(
|
||||
backend=backend,
|
||||
layer_names=["main"],
|
||||
kv_cache_spec=spec,
|
||||
kv_cache_group_id=0,
|
||||
)
|
||||
|
||||
|
||||
def test_main_cache_byte_identical_through_production_allocator():
|
||||
"""AC-2: drive the real allocator (`_reshape_kv_cache`) for the M3 main
|
||||
"""AC-2: drive the real allocator (`create_kv_cache_views`) for the M3 main
|
||||
`FullAttentionSpec` under HND and assert the backend-visible view has the
|
||||
same shape, stride, and storage offset as the packed-HND allocation; the
|
||||
indexer `MLAAttentionSpec` allocates through the same path to its 3-dim
|
||||
shape."""
|
||||
same shape, stride, and storage offset as the packed-HND allocation."""
|
||||
nb = 4
|
||||
spec = FullAttentionSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=NUM_KV_HEADS,
|
||||
head_size=HEAD_DIM,
|
||||
head_size_v=HEAD_DIM,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
spec = _main_spec()
|
||||
raw = torch.zeros(nb * spec.page_size_bytes, dtype=torch.int8)
|
||||
group = _make_attn_group(MiniMaxM3SparseBackend, spec)
|
||||
try:
|
||||
set_kv_cache_layout("HND")
|
||||
kv_caches = _reshape_kv_cache([group], {"main": raw}, "auto", [BLOCK_SIZE], {})
|
||||
finally:
|
||||
set_kv_cache_layout(None)
|
||||
view = kv_caches["main"]
|
||||
view = dense_kv_cache_views(raw, spec, nb, 1, KVCacheLayout.LBHNC, BLOCK_SIZE)[0]
|
||||
|
||||
oracle = raw.view(DTYPE).view((nb, NUM_KV_HEADS, BLOCK_SIZE, 2 * HEAD_DIM))
|
||||
assert tuple(view.shape) == tuple(oracle.shape)
|
||||
assert view.stride() == oracle.stride()
|
||||
assert view.storage_offset() == oracle.storage_offset()
|
||||
|
||||
# Indexer cache allocates through the same path under both layouts.
|
||||
ispec = MLAAttentionSpec(
|
||||
block_size=BLOCK_SIZE, num_kv_heads=1, head_size=HEAD_DIM, dtype=DTYPE
|
||||
)
|
||||
for layout in ("NHD", "HND"):
|
||||
iraw = torch.zeros(nb * ispec.page_size_bytes, dtype=torch.int8)
|
||||
igroup = AttentionGroup(
|
||||
backend=MiniMaxM3IndexerBackend,
|
||||
layer_names=["idx"],
|
||||
kv_cache_spec=ispec,
|
||||
kv_cache_group_id=0,
|
||||
)
|
||||
try:
|
||||
set_kv_cache_layout(layout)
|
||||
iout = _reshape_kv_cache([igroup], {"idx": iraw}, "auto", [BLOCK_SIZE], {})
|
||||
finally:
|
||||
set_kv_cache_layout(None)
|
||||
assert tuple(iout["idx"].shape) == (nb, BLOCK_SIZE, HEAD_DIM)
|
||||
|
||||
|
||||
def test_indexer_inherited_stride_order_trips_allocator_assert():
|
||||
"""AC-4 negative: without the indexer override, the inherited 4-element
|
||||
stride order trips the allocator's `len(stride_order) == len(shape)` assert
|
||||
for the 3-dim indexer shape; the `AssertionError` is NOT swallowed by the
|
||||
allocator's `(AttributeError, NotImplementedError)` fallback."""
|
||||
|
||||
class _BrokenIndexerBackend(MiniMaxM3IndexerBackend):
|
||||
# Simulate inheriting the parent's 4-element stride order.
|
||||
get_kv_cache_stride_order = staticmethod(
|
||||
MiniMaxM3SparseBackend.get_kv_cache_stride_order
|
||||
)
|
||||
|
||||
nb = 4
|
||||
ispec = MLAAttentionSpec(
|
||||
block_size=BLOCK_SIZE, num_kv_heads=1, head_size=HEAD_DIM, dtype=DTYPE
|
||||
)
|
||||
iraw = torch.zeros(nb * ispec.page_size_bytes, dtype=torch.int8)
|
||||
igroup = AttentionGroup(
|
||||
backend=_BrokenIndexerBackend,
|
||||
layer_names=["idx"],
|
||||
kv_cache_spec=ispec,
|
||||
kv_cache_group_id=0,
|
||||
)
|
||||
try:
|
||||
set_kv_cache_layout("HND")
|
||||
with pytest.raises(AssertionError):
|
||||
_reshape_kv_cache([igroup], {"idx": iraw}, "auto", [BLOCK_SIZE], {})
|
||||
finally:
|
||||
set_kv_cache_layout(None)
|
||||
|
||||
|
||||
def test_padded_main_cache_is_flagged():
|
||||
"""AC-2.1 negative: the M3 main cache relies on the allocator's
|
||||
@@ -1465,11 +1360,7 @@ def test_padded_main_cache_is_flagged():
|
||||
)
|
||||
assert inv_order[0] == 0, "main GQA cache must remain block-first"
|
||||
|
||||
try:
|
||||
set_kv_cache_layout("HND")
|
||||
stride_order = MiniMaxM3SparseBackend.get_kv_cache_stride_order()
|
||||
finally:
|
||||
set_kv_cache_layout(None)
|
||||
stride_order = KVCacheLayout.LBHNC.layer_view_order
|
||||
|
||||
good = FullAttentionSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
@@ -1493,13 +1384,13 @@ def test_padded_main_cache_is_flagged():
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_layout", ["NHD", "HND"], indirect=True)
|
||||
def test_reshape_and_cache_flash_write_persists(kv_layout: str):
|
||||
def test_reshape_and_cache_flash_write_persists(kv_layout: KVCacheLayout):
|
||||
"""AC-5 write path: the `reshape_and_cache_flash` write site now consumes
|
||||
packed-content K/V split views. Writing through those views must persist
|
||||
into the bound storage under both layouts."""
|
||||
torch.manual_seed(0)
|
||||
num_pages = 4
|
||||
kv_cache = _allocate_main_kv_via_contract(num_pages)
|
||||
kv_cache = _allocate_main_kv_via_contract(num_pages, kv_layout)
|
||||
with torch.no_grad():
|
||||
kv_cache.zero_()
|
||||
|
||||
|
||||
@@ -310,18 +310,23 @@ def test_aiter_mha_backend_contract():
|
||||
|
||||
|
||||
def test_aiter_mha_backend_validates_kv_cache_block_size():
|
||||
"""The backend should reject KV cache shapes that cannot be gathered
|
||||
"""The backend should reject KV cache specs that cannot be gathered
|
||||
correctly by the ROCm kernel."""
|
||||
from vllm.v1.attention.backends.rocm_aiter_fa import AiterFlashAttentionBackend
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec
|
||||
|
||||
assert AiterFlashAttentionBackend.get_kv_cache_shape(8, 16, 8, 128) == (
|
||||
8,
|
||||
8,
|
||||
16,
|
||||
256,
|
||||
)
|
||||
with pytest.raises(ValueError, match="Block size must be a multiple of 16"):
|
||||
AiterFlashAttentionBackend.get_kv_cache_shape(8, 15, 8, 128)
|
||||
AiterFlashAttentionBackend.customize_spec(
|
||||
FullAttentionSpec(
|
||||
block_size=15, num_kv_heads=8, head_size=128, dtype=torch.bfloat16
|
||||
)
|
||||
)
|
||||
# block_size == 1 is the per-token page-size probe and must pass through.
|
||||
AiterFlashAttentionBackend.customize_spec(
|
||||
FullAttentionSpec(
|
||||
block_size=1, num_kv_heads=8, head_size=128, dtype=torch.bfloat16
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_aiter_mha_backend_supports_compute_capability_matches_mi3xx_probe():
|
||||
|
||||
@@ -11,6 +11,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.kv_cache_interface import KVCacheLayout
|
||||
|
||||
if current_platform.is_rocm():
|
||||
pytest.skip(
|
||||
@@ -34,51 +35,40 @@ def to_float8(x, dtype=None):
|
||||
return x_scl_sat.to(dtype), scale.float().reciprocal()
|
||||
|
||||
|
||||
def make_contiguous_kv_cache(num_blocks, num_kv_heads, block_size, head_size):
|
||||
"""Create a standard contiguous fp8 KV cache (HND layout)."""
|
||||
raw = torch.randn(
|
||||
num_blocks,
|
||||
2,
|
||||
num_kv_heads,
|
||||
block_size,
|
||||
head_size,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
)
|
||||
kv_cache, scale = to_float8(raw)
|
||||
return kv_cache, scale
|
||||
|
||||
|
||||
def make_cross_layer_kv_cache(
|
||||
def make_random_kv_cache(
|
||||
num_blocks,
|
||||
num_kv_heads,
|
||||
block_size,
|
||||
head_size,
|
||||
num_layers=4,
|
||||
layout=KVCacheLayout.LBHNC,
|
||||
num_layers=None,
|
||||
):
|
||||
"""
|
||||
Create a non-contiguous per-layer view mimicking cross-layer allocation.
|
||||
Create a random fp8 KV cache mimicking the production allocation.
|
||||
|
||||
Physical layout: (num_blocks, 2, num_kv_heads, num_layers, block_size, head_size)
|
||||
Physical layout: [L, B, H, N, 2 * hs] permuted by ``layout.stride_order``
|
||||
Returned view: (num_blocks, 2, num_kv_heads, block_size, head_size)
|
||||
with non-contiguous strides on dims 0, 1, 2 (they skip over num_layers).
|
||||
with non-contiguous strides under cross-layer layouts (they skip over
|
||||
num_layers), matching the actual forward path.
|
||||
"""
|
||||
raw = torch.randn(
|
||||
logical_4d = (num_blocks, num_kv_heads, block_size, 2 * head_size)
|
||||
if num_layers is None:
|
||||
num_layers = 1 if layout.is_layer_compact else 2
|
||||
logical_5d = (num_layers, *logical_4d)
|
||||
physical_5d = tuple(logical_5d[i] for i in layout.stride_order)
|
||||
inv_order = [layout.stride_order.index(i) for i in range(5)]
|
||||
|
||||
raw_phys = torch.randn(*physical_5d, dtype=torch.bfloat16, device="cuda")
|
||||
fp8_phys, scale = to_float8(raw_phys)
|
||||
fp8_4d = fp8_phys.permute(*inv_order)[0]
|
||||
kv_5d = fp8_4d.view(
|
||||
num_blocks,
|
||||
2,
|
||||
num_kv_heads,
|
||||
num_layers,
|
||||
block_size,
|
||||
2,
|
||||
head_size,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
)
|
||||
fp8_full, scale = to_float8(raw)
|
||||
layer_view = fp8_full[:, :, :, 0, :, :]
|
||||
assert not layer_view.is_contiguous(), (
|
||||
f"Expected non-contiguous view, got strides {layer_view.stride()}"
|
||||
)
|
||||
return layer_view, scale
|
||||
).permute(0, 3, 1, 2, 4)
|
||||
return kv_5d, scale
|
||||
|
||||
|
||||
def ref_dequant(kv_cache, block_tables, k_scale, v_scale, dequant_dtype):
|
||||
@@ -114,7 +104,7 @@ def ref_dequant(kv_cache, block_tables, k_scale, v_scale, dequant_dtype):
|
||||
@pytest.mark.parametrize("block_size", [16, 32])
|
||||
@pytest.mark.parametrize("batch_size", [1, 4])
|
||||
@pytest.mark.parametrize("num_pages_per_seq", [3, 8])
|
||||
@pytest.mark.parametrize("contiguous", [True, False])
|
||||
@pytest.mark.parametrize("layout", list(KVCacheLayout))
|
||||
@torch.inference_mode()
|
||||
def test_trtllm_kvfp8_dequant(
|
||||
num_kv_heads: int,
|
||||
@@ -122,7 +112,7 @@ def test_trtllm_kvfp8_dequant(
|
||||
block_size: int,
|
||||
batch_size: int,
|
||||
num_pages_per_seq: int,
|
||||
contiguous: bool,
|
||||
layout: KVCacheLayout,
|
||||
):
|
||||
from vllm.v1.attention.backends.flashinfer import (
|
||||
trtllm_prefill_attn_kvfp8_dequant,
|
||||
@@ -130,20 +120,13 @@ def test_trtllm_kvfp8_dequant(
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
|
||||
if contiguous:
|
||||
kv_cache, scale = make_contiguous_kv_cache(
|
||||
NUM_BLOCKS,
|
||||
num_kv_heads,
|
||||
block_size,
|
||||
head_size,
|
||||
)
|
||||
else:
|
||||
kv_cache, scale = make_cross_layer_kv_cache(
|
||||
NUM_BLOCKS,
|
||||
num_kv_heads,
|
||||
block_size,
|
||||
head_size,
|
||||
)
|
||||
kv_cache, scale = make_random_kv_cache(
|
||||
NUM_BLOCKS,
|
||||
num_kv_heads,
|
||||
block_size,
|
||||
head_size,
|
||||
layout=layout,
|
||||
)
|
||||
|
||||
k_scale = scale.clone()
|
||||
v_scale = scale.clone()
|
||||
@@ -187,7 +170,7 @@ def test_block_tables_with_zero_pages():
|
||||
torch.set_default_device("cuda")
|
||||
num_kv_heads, block_size, head_size = 8, 16, 64
|
||||
|
||||
kv_cache, scale = make_contiguous_kv_cache(
|
||||
kv_cache, scale = make_random_kv_cache(
|
||||
NUM_BLOCKS,
|
||||
num_kv_heads,
|
||||
block_size,
|
||||
@@ -234,7 +217,7 @@ def test_all_zero_block_tables():
|
||||
torch.set_default_device("cuda")
|
||||
num_kv_heads, block_size, head_size = 4, 16, 64
|
||||
|
||||
kv_cache, scale = make_contiguous_kv_cache(
|
||||
kv_cache, scale = make_random_kv_cache(
|
||||
NUM_BLOCKS,
|
||||
num_kv_heads,
|
||||
block_size,
|
||||
@@ -266,7 +249,7 @@ def test_different_k_v_scales():
|
||||
torch.set_default_device("cuda")
|
||||
num_kv_heads, block_size, head_size = 8, 16, 64
|
||||
|
||||
kv_cache, _ = make_contiguous_kv_cache(
|
||||
kv_cache, _ = make_random_kv_cache(
|
||||
NUM_BLOCKS,
|
||||
num_kv_heads,
|
||||
block_size,
|
||||
@@ -299,7 +282,7 @@ def test_single_page_per_seq():
|
||||
torch.set_default_device("cuda")
|
||||
num_kv_heads, block_size, head_size = 8, 16, 128
|
||||
|
||||
kv_cache, scale = make_contiguous_kv_cache(
|
||||
kv_cache, scale = make_random_kv_cache(
|
||||
NUM_BLOCKS,
|
||||
num_kv_heads,
|
||||
block_size,
|
||||
@@ -332,7 +315,7 @@ def test_large_page_indices():
|
||||
num_kv_heads, block_size, head_size = 8, 16, 128
|
||||
large_num_blocks = 32768
|
||||
|
||||
kv_cache, scale = make_contiguous_kv_cache(
|
||||
kv_cache, scale = make_random_kv_cache(
|
||||
large_num_blocks,
|
||||
num_kv_heads,
|
||||
block_size,
|
||||
@@ -369,7 +352,7 @@ def test_large_block_size():
|
||||
torch.set_default_device("cuda")
|
||||
num_kv_heads, block_size, head_size = 4, 64, 128
|
||||
|
||||
kv_cache, scale = make_contiguous_kv_cache(
|
||||
kv_cache, scale = make_random_kv_cache(
|
||||
NUM_BLOCKS,
|
||||
num_kv_heads,
|
||||
block_size,
|
||||
@@ -401,7 +384,7 @@ def test_large_block_size():
|
||||
def test_cross_layer_many_layers():
|
||||
"""
|
||||
Non-contiguous with 36 layers -- matches real gpt-oss-120b.
|
||||
Strides are far from contiguous (factor of 36 in the gaps).
|
||||
Cross-layer (BLHNC) strides are far from contiguous (factor of 36 in the gaps).
|
||||
"""
|
||||
from vllm.v1.attention.backends.flashinfer import (
|
||||
trtllm_prefill_attn_kvfp8_dequant,
|
||||
@@ -409,14 +392,14 @@ def test_cross_layer_many_layers():
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
num_kv_heads, block_size, head_size = 8, 16, 64
|
||||
num_layers = 36
|
||||
|
||||
kv_cache, scale = make_cross_layer_kv_cache(
|
||||
kv_cache, scale = make_random_kv_cache(
|
||||
NUM_BLOCKS,
|
||||
num_kv_heads,
|
||||
block_size,
|
||||
head_size,
|
||||
num_layers=num_layers,
|
||||
layout=KVCacheLayout.BLHNC,
|
||||
num_layers=36,
|
||||
)
|
||||
k_scale = v_scale = scale.clone()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user