[Attention] Make FlexAttention and FlashAttention use num-blocks first layouts (#42095)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
Lucas Wilkinson
2026-05-26 19:55:56 -07:00
committed by GitHub
co-authored by Matthew Bonanni Nicolò Lucchesi
parent d8eebe6d97
commit 7e33081cee
20 changed files with 209 additions and 389 deletions
+30 -19
View File
@@ -136,7 +136,8 @@ def create_and_prepopulate_kv_cache(
block_table = common_attn_metadata.block_table_tensor
slot_mapping = common_attn_metadata.slot_mapping
# Create KV cache
# Create KV cache and populate in (2, num_blocks, ...) layout for easy
# flat indexing, then transpose to (num_blocks, 2, ...) layout.
kv_cache = torch.zeros(
2, num_blocks, block_size, num_kv_heads, head_size, dtype=dtype, device=device
)
@@ -155,6 +156,9 @@ def create_and_prepopulate_kv_cache(
# Stay block aligned and allocate enough blocks for the new tokens
start_block_idx += cdiv(int(seq_lens[i]), block_size)
# Transpose to (num_blocks, 2, ...) layout
kv_cache = kv_cache.transpose(0, 1).contiguous()
blocks_end = start_block_idx
# Permute the context blocks (excluding block 0 which is null)
@@ -168,7 +172,7 @@ def create_and_prepopulate_kv_cache(
inv_perm = torch.zeros(blocks_end, dtype=torch.long, device=device)
# Add 1 to account for starting from block 1
inv_perm[1:] = torch.argsort(perm) + 1
kv_cache[:, 1:blocks_end, ...] = kv_cache[:, perm, ...]
kv_cache[1:blocks_end, ...] = kv_cache[perm, ...]
# Construct the right block table
# Start from block_id=1 since block_id=0 is considered the null block
@@ -473,28 +477,35 @@ def _test_backend_correctness(
# Note: flex_attention has known Triton kernel compatibility issues
# with test infrastructures
for backend_name in backend_to_test:
# FlashAttentionm + FlexAttention:
# [2, num_blocks, block_size, num_kv_heads, head_size]
# FlashInfer + Triton:
# [num_blocks, 2, block_size, num_kv_heads, head_size]
# Select the appropriate KV cache format for each backend
kv_cache_for_backend = kv_cache
reset_kv_cache_layout = False
if backend_name in (
AttentionBackendEnum.FLASHINFER,
AttentionBackendEnum.TRITON_ATTN,
):
kv_cache_for_backend = kv_cache.transpose(0, 1)
# Resolve backend class for both enum and string names.
actual_backend = backend_name
if backend_name == "FLEX_ATTENTION_SLOW":
actual_backend = AttentionBackendEnum.FLEX_ATTENTION
if hasattr(actual_backend, "get_class"):
backend_cls = actual_backend.get_class()
else:
backend_cls = None
if backend_name == AttentionBackendEnum.FLASHINFER:
# For FlashInfer default to HND layout and
kv_cache_for_backend = (
kv_cache_for_backend.transpose(2, 3).contiguous().transpose(2, 3)
)
set_kv_cache_layout("HND")
reset_kv_cache_layout = True
elif backend_name == AttentionBackendEnum.TRITON_ATTN:
kv_cache_for_backend = kv_cache_for_backend.contiguous()
# Apply stride order like runtime does in
# _reshape_kv_cache (attn_utils.py:182-210): permute to physical
# layout, make contiguous, then permute to logical layout.
kv_cache_for_backend = kv_cache
if backend_cls is not None:
try:
stride_order = backend_cls.get_kv_cache_stride_order()
except (AttributeError, NotImplementedError):
stride_order = tuple(range(kv_cache.ndim))
if stride_order != tuple(range(kv_cache.ndim)):
inv_order = [stride_order.index(i) for i in range(len(stride_order))]
kv_cache_for_backend = (
kv_cache.permute(*stride_order).contiguous().permute(*inv_order)
)
try:
backend_output = run_attention_backend(
@@ -19,7 +19,7 @@ size-1 dimensions via torch.as_strided — zero-copy.
The degenerate stride manifests at different positions in different backends:
- FlashInfer: stride(-3) after kv_cache.permute() → shape [..., 1, B, D]
- FlashAttention: stride(-2) after kv_cache.unbind(0) → shape [N, B, 1, D]
- FlashAttention: stride(-2) after kv_cache.unbind(1) → shape [N, B, 1, D]
"""
import torch
@@ -1,14 +1,13 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections import defaultdict
from unittest.mock import MagicMock, patch
from unittest.mock import MagicMock
import pytest
import torch
from vllm.platforms import current_platform
from vllm.utils.torch_utils import get_dtype_size
from vllm.v1.attention.backend import AttentionBackend
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.attention.backends.utils import set_kv_cache_layout
from vllm.v1.kv_cache_interface import (
@@ -85,15 +84,6 @@ def _allocate_and_reshape_kv_caches(
set_kv_cache_layout(None)
def _make_mock_layer(backend_cls: type[AttentionBackend]):
"""
Create a mock AttentionLayerBase whose get_attn_backend returns backend_cls.
"""
layer = MagicMock()
layer.get_attn_backend.return_value = backend_cls
return layer
def _make_worker(kv_cache_config: KVCacheConfig):
"""
Create an OffloadingConnectorWorker with mocked dependencies.
@@ -119,11 +109,7 @@ def _make_worker(kv_cache_config: KVCacheConfig):
@pytest.mark.parametrize("backend", ATTN_BACKENDS)
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.offloading"
".worker.get_layers_from_vllm_config"
)
def test_register_kv_caches(mock_get_layers, backend):
def test_register_kv_caches(backend):
"""Test register_kv_caches with multiple groups covering all layer types.
Creates one FullAttention group, one MLA group, one Mamba group, and
@@ -287,13 +273,6 @@ def test_register_kv_caches(mock_get_layers, backend):
device=torch.device("cuda:0"),
)
mock_layers: dict[str, MagicMock] = {}
for layer_name in attn_layer_names:
mock_layers[layer_name] = _make_mock_layer(backend_cls)
for layer_name in mla_layer_names:
mock_layers[layer_name] = _make_mock_layer(DeepseekV32IndexerBackend)
mock_get_layers.return_value = mock_layers
worker, spec = _make_worker(kv_cache_config)
worker.register_kv_caches(kv_caches)
@@ -360,11 +339,7 @@ def test_register_kv_caches(mock_get_layers, backend):
@pytest.mark.parametrize("backend", ATTN_BACKENDS)
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.offloading"
".worker.get_layers_from_vllm_config"
)
def test_register_kv_caches_uniform_type(mock_get_layers, backend):
def test_register_kv_caches_uniform_type(backend):
"""Test register_kv_caches with UniformTypeKVCacheSpecs.
Two attention layers use the same backend but different num_kv_heads,
@@ -441,64 +416,29 @@ def test_register_kv_caches_uniform_type(mock_get_layers, backend):
device=torch.device("cuda:0"),
)
mock_get_layers.return_value = {
layer_a: _make_mock_layer(backend_cls),
layer_b: _make_mock_layer(backend_cls),
}
worker, spec = _make_worker(kv_cache_config)
worker.register_kv_caches(kv_caches)
canonical = spec.get_handlers.call_args[0][0]
assert isinstance(canonical, CanonicalKVCaches)
unbinds = backend_cls.get_name() in ("FLASH_ATTN", "FLEX_ATTENTION")
tensors_per_layer = 2 if unbinds else 1
for block_tensor in canonical.tensors:
assert block_tensor.tensor.dtype == torch.int8
# Single group with refs from both layers
assert len(canonical.group_data_refs) == 1
group_refs = canonical.group_data_refs[0]
assert len(group_refs) == 2 * tensors_per_layer
assert len(group_refs) == 2
if unbinds:
half_a = spec_a.page_size_bytes // 2
half_b = spec_b.page_size_bytes // 2
assert len(canonical.tensors) == 2
assert canonical.tensors[0].page_size_bytes == spec_a.page_size_bytes
assert canonical.tensors[1].page_size_bytes == spec_b.page_size_bytes
assert canonical.tensors[0].tensor.shape == (NUM_BLOCKS, spec_a.page_size_bytes)
assert canonical.tensors[1].tensor.shape == (NUM_BLOCKS, spec_b.page_size_bytes)
assert len(canonical.tensors) == 4
assert canonical.tensors[0].page_size_bytes == half_a
assert canonical.tensors[1].page_size_bytes == half_a
assert canonical.tensors[2].page_size_bytes == half_b
assert canonical.tensors[3].page_size_bytes == half_b
assert canonical.tensors[0].tensor.shape == (NUM_BLOCKS, half_a)
assert canonical.tensors[1].tensor.shape == (NUM_BLOCKS, half_a)
assert canonical.tensors[2].tensor.shape == (NUM_BLOCKS, half_b)
assert canonical.tensors[3].tensor.shape == (NUM_BLOCKS, half_b)
assert group_refs[0] == CanonicalKVCacheRef(
tensor_idx=0, page_size_bytes=half_a
)
assert group_refs[1] == CanonicalKVCacheRef(
tensor_idx=1, page_size_bytes=half_a
)
assert group_refs[2] == CanonicalKVCacheRef(
tensor_idx=2, page_size_bytes=half_b
)
assert group_refs[3] == CanonicalKVCacheRef(
tensor_idx=3, page_size_bytes=half_b
)
else:
assert len(canonical.tensors) == 2
assert canonical.tensors[0].page_size_bytes == spec_a.page_size_bytes
assert canonical.tensors[1].page_size_bytes == spec_b.page_size_bytes
assert canonical.tensors[0].tensor.shape == (NUM_BLOCKS, spec_a.page_size_bytes)
assert canonical.tensors[1].tensor.shape == (NUM_BLOCKS, spec_b.page_size_bytes)
assert group_refs[0] == CanonicalKVCacheRef(
tensor_idx=0, page_size_bytes=spec_a.page_size_bytes
)
assert group_refs[1] == CanonicalKVCacheRef(
tensor_idx=1, page_size_bytes=spec_b.page_size_bytes
)
assert group_refs[0] == CanonicalKVCacheRef(
tensor_idx=0, page_size_bytes=spec_a.page_size_bytes
)
assert group_refs[1] == CanonicalKVCacheRef(
tensor_idx=1, page_size_bytes=spec_b.page_size_bytes
)
@@ -3,7 +3,7 @@
from collections.abc import Iterable, Iterator
from dataclasses import dataclass
from typing import Any
from unittest.mock import MagicMock, patch
from unittest.mock import MagicMock
import pytest
import torch
@@ -25,7 +25,6 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading_connector import (
)
from vllm.forward_context import ForwardContext
from vllm.utils.hashing import sha256
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
from vllm.v1.core.kv_cache_utils import (
get_request_block_hasher,
init_none_hash,
@@ -239,37 +238,22 @@ class RequestRunner:
# register worker kv_caches to enable OffloadingWorker creations
# set_current_vllm_config is needed for get_kv_cache_layout() to work
# Mock get_layers_from_vllm_config so that mock layer names
# resolve to layers whose get_attn_backend() returns
# FlashAttentionBackend.
def _mock_get_layers(_vllm_config, _layer_type, layer_names):
mock_layer = MagicMock()
mock_layer.get_attn_backend.return_value = FlashAttentionBackend
return {name: mock_layer for name in layer_names}
kv_caches: dict[str, torch.Tensor] = {}
for group in kv_cache_groups:
spec = group.kv_cache_spec
for layer_name in group.layer_names:
# Shape follows FlashAttention layout:
# (2, num_blocks, block_size, num_kv_heads, head_size)
# Shape: (num_blocks, 2, block_size, num_kv_heads, head_size)
kv_caches[layer_name] = torch.empty(
2,
num_gpu_blocks,
2,
spec.block_size,
spec.num_kv_heads,
spec.head_size,
dtype=spec.dtype,
)
with (
set_current_vllm_config(vllm_config),
patch(
"vllm.distributed.kv_transfer.kv_connector.v1"
".offloading.worker.get_layers_from_vllm_config",
side_effect=_mock_get_layers,
),
):
with set_current_vllm_config(vllm_config):
self.worker_connector.register_kv_caches(kv_caches)
# extract connector of scheduler
@@ -369,15 +369,31 @@ async def test_kv_producer(monkeypatch):
with patch.object(
prefill_worker, "_send_blocks", return_value=0
) as mock_send_blocks:
# With blocks-first layout, each block is virtually split
# into K and V halves, producing non-coalesced transfers.
kv_half = block_len // 2
def expected_split_transfers(src_base, dst_base, src_blocks, dst_blocks):
"""Build expected (src_ptrs, dst_ptrs, lengths) for
virtual-split K/V transfers."""
src_ptrs, dst_ptrs, lengths = [], [], []
for kv_offset in (0, kv_half):
for sb, db in zip(src_blocks, dst_blocks):
src_ptrs.append(src_base + sb * block_len + kv_offset)
dst_ptrs.append(dst_base + db * block_len + kv_offset)
lengths.append(kv_half)
return src_ptrs, dst_ptrs, lengths
# Normal case: 2 blocks to 2 blocks
# Worker processes the consumer's request
await prefill_worker.send_kv_to_decode(identity, mock_socket, xfer_meta)
# Verify transfer parameters are correct
src_ptr = 0x1000 + 10 * block_len
dst_ptr = 0x2000 + 20 * block_len
length = 2 * block_len
src, dst, lens = expected_split_transfers(
0x1000, 0x2000, [10, 11], [20, 21]
)
mock_send_blocks.assert_called_once_with(
"consumer-host:54321", [src_ptr], [dst_ptr], [length]
"consumer-host:54321",
src,
dst,
lens,
)
mock_socket.send_multipart.assert_called_once()
@@ -404,11 +420,12 @@ async def test_kv_producer(monkeypatch):
# Worker processes the consumer's request
await prefill_worker.send_kv_to_decode(identity, mock_socket, xfer_meta)
# Verify transfer parameters are correct: 11 to 20
src_ptr = 0x1000 + 11 * block_len
dst_ptr = 0x2000 + 20 * block_len
length = 1 * block_len
src, dst, lens = expected_split_transfers(0x1000, 0x2000, [11], [20])
mock_send_blocks.assert_called_once_with(
"consumer-host:54321", [src_ptr], [dst_ptr], [length]
"consumer-host:54321",
src,
dst,
lens,
)
mock_socket.send_multipart.assert_called_once()
@@ -618,18 +635,14 @@ def test_register_kv_caches():
mock_batch_register.assert_called_once()
registered_ptrs, registered_lens = mock_batch_register.call_args[0]
expected_ptrs = {
tensor.data_ptr()
for kv_pair in kv_caches.values()
for tensor in kv_pair
}
expected_ptrs = {tensor.data_ptr() for tensor in kv_caches.values()}
assert set(registered_ptrs) == expected_ptrs
assert set(registered_lens) == {tensor1[0].nbytes}
assert set(registered_lens) == {tensor1.nbytes}
# Verify block_len_per_layer is set correctly.
assert len(worker.block_len_per_layer) == len(registered_ptrs)
for bl in worker.block_len_per_layer:
assert bl == tensor1[0].nbytes // tensor1.shape[1]
assert bl == tensor1.nbytes // tensor1.shape[0]
def test_register_kv_caches_supports_mixed_mla_and_eagle_shapes():
@@ -791,33 +804,49 @@ async def test_kv_producer_heterogeneous_tp(monkeypatch, d_tp_size):
# Flatten nested per-group block IDs for assertions
flat_local = [b for g in local_block_ids for b in g]
flat_remote = [b for g in remote_block_ids for b in g]
num_blocks = len(flat_local)
# Heterogeneous TP: blocks cannot be coalesced because
# local and remote block_lens differ
assert len(src_ptrs) == len(flat_local)
assert len(dst_ptrs) == len(flat_local)
assert len(lengths) == len(flat_local)
# With blocks-first layout, virtual split halves block
# lengths and doubles transfer regions (K + V).
local_kv_block_len = local_block_len // 2
remote_kv_block_len = remote_block_len // 2
# Compute expected offsets based on TP ratio
assert len(src_ptrs) == 2 * num_blocks
assert len(dst_ptrs) == 2 * num_blocks
assert len(lengths) == 2 * num_blocks
# Compute expected offsets using kv_block_len
if d_tp_size <= P_TP_SIZE:
tp_ratio = P_TP_SIZE // d_tp_size
expected_src_off = 0
expected_dst_off = (P_TP_RANK % tp_ratio) * local_block_len
expected_xfer_len = local_block_len
expected_dst_off = (P_TP_RANK % tp_ratio) * local_kv_block_len
expected_xfer_len = local_kv_block_len
else:
ratio_abs = d_tp_size // P_TP_SIZE
expected_src_off = (d_rank % ratio_abs) * remote_block_len
expected_src_off = (d_rank % ratio_abs) * remote_kv_block_len
expected_dst_off = 0
expected_xfer_len = remote_block_len
expected_xfer_len = remote_kv_block_len
for idx, (lblk, rblk) in enumerate(zip(flat_local, flat_remote)):
assert src_ptrs[idx] == (
0x1000 + lblk * local_block_len + expected_src_off
)
assert dst_ptrs[idx] == (
0x2000 + rblk * remote_block_len + expected_dst_off
)
assert lengths[idx] == expected_xfer_len
# First num_blocks entries are K region,
# next num_blocks are V region.
for region_idx in range(2):
local_region_base = 0x1000 + region_idx * local_kv_block_len
remote_region_base = 0x2000 + region_idx * remote_kv_block_len
for blk_idx, (lblk, rblk) in enumerate(
zip(flat_local, flat_remote)
):
idx = region_idx * num_blocks + blk_idx
assert src_ptrs[idx] == (
local_region_base
+ lblk * local_block_len
+ expected_src_off
)
assert dst_ptrs[idx] == (
remote_region_base
+ rblk * remote_block_len
+ expected_dst_off
)
assert lengths[idx] == expected_xfer_len
# Verify successful response sent back to consumer
mock_socket.send_multipart.assert_called_once()
@@ -1635,6 +1635,7 @@ def test_register_kv_caches(
num_blocks=1, block_size=16, num_kv_heads=1, head_size=1
)
is_blocks_first = len(test_shape) == 5 and test_shape[0] == 1
virtually_split = is_blocks_first and not connector.prefer_cross_layer_blocks
if connector.prefer_cross_layer_blocks:
with set_current_vllm_config(vllm_config):
@@ -1665,7 +1666,7 @@ def test_register_kv_caches(
]
expected_num_entries = 1
expected_blocks_count = num_blocks * (2 if is_blocks_first else 1)
expected_blocks_count = num_blocks * (2 if virtually_split else 1)
kv_caches = {"all-layers": cross_layers_kv_cache}
else:
@@ -1739,7 +1740,7 @@ def test_register_kv_caches(
else:
num_blocks = kv_cache_config.num_blocks
if is_blocks_first:
if virtually_split:
expected_block_len = expected_tensor_size // num_blocks // 2
else:
expected_block_len = expected_tensor_size // num_blocks
+1 -29
View File
@@ -34,7 +34,6 @@ from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.core.kv_cache_utils import estimate_max_model_len, get_kv_cache_configs
from vllm.v1.core.sched.output import CachedRequestData, NewRequestData, SchedulerOutput
from vllm.v1.kv_cache_interface import (
AttentionSpec,
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
@@ -44,7 +43,7 @@ from vllm.v1.sample.metadata import SamplingMetadata
from vllm.v1.spec_decode.metadata import SpecDecodeMetadata
from vllm.v1.worker.gpu_input_batch import InputBatch
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
from vllm.v1.worker.utils import AttentionGroup, select_common_block_size
from vllm.v1.worker.utils import select_common_block_size
BLOCK_SIZE = 16
NUM_BLOCKS = 10
@@ -1195,33 +1194,6 @@ def test_hybrid_attention_mamba_tensor_shapes():
assert torch.equal(actual_ssm, expected_ssm)
def test_update_hybrid_attention_mamba_layout_with_num_block_2_rewrites_stride():
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
ambiguous_cache = torch.empty((2, 2, BLOCK_SIZE, 1, 8), dtype=torch.float16)
"""Ambiguous, because both dims[0=kv_dim] and dims[1=num_blocks] == 2"""
hidden_size = ambiguous_cache.shape[2:].numel()
assert ambiguous_cache.stride()[:2] == (2 * hidden_size, hidden_size)
attention_spec = AttentionSpec(
block_size=BLOCK_SIZE, num_kv_heads=1, head_size=8, dtype=torch.float16
)
runner_stub = SimpleNamespace(
cache_config=SimpleNamespace(cache_dtype="auto"),
_kv_cache_spec_attn_group_iterator=lambda: iter(
[AttentionGroup(FlashAttentionBackend, ["attn"], attention_spec, 0)]
),
)
GPUModelRunner._update_hybrid_attention_mamba_layout(
runner_stub, {"attn": ambiguous_cache}, [BLOCK_SIZE]
)
assert ambiguous_cache.stride()[:2] == (hidden_size, 2 * hidden_size), """\
We expect _update_hybrid_attention_mamba_layout to re-stride the cache from:
(2, num_blocks) -> (num_blocks, 2), even when num_blocks==2,
which was ambiguous before get_kv_cache_block_dim was used"""
def test_hybrid_block_table_initialization():
"""Test hybrid block table with different kernel and kvcache_manager block
sizes."""