[Attention] Make FlexAttention and FlashAttention use num-blocks first layouts (#42095)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
co-authored by
Matthew Bonanni
Nicolò Lucchesi
parent
d8eebe6d97
commit
7e33081cee
@@ -136,7 +136,8 @@ def create_and_prepopulate_kv_cache(
|
||||
block_table = common_attn_metadata.block_table_tensor
|
||||
slot_mapping = common_attn_metadata.slot_mapping
|
||||
|
||||
# Create KV cache
|
||||
# Create KV cache and populate in (2, num_blocks, ...) layout for easy
|
||||
# flat indexing, then transpose to (num_blocks, 2, ...) layout.
|
||||
kv_cache = torch.zeros(
|
||||
2, num_blocks, block_size, num_kv_heads, head_size, dtype=dtype, device=device
|
||||
)
|
||||
@@ -155,6 +156,9 @@ def create_and_prepopulate_kv_cache(
|
||||
# Stay block aligned and allocate enough blocks for the new tokens
|
||||
start_block_idx += cdiv(int(seq_lens[i]), block_size)
|
||||
|
||||
# Transpose to (num_blocks, 2, ...) layout
|
||||
kv_cache = kv_cache.transpose(0, 1).contiguous()
|
||||
|
||||
blocks_end = start_block_idx
|
||||
|
||||
# Permute the context blocks (excluding block 0 which is null)
|
||||
@@ -168,7 +172,7 @@ def create_and_prepopulate_kv_cache(
|
||||
inv_perm = torch.zeros(blocks_end, dtype=torch.long, device=device)
|
||||
# Add 1 to account for starting from block 1
|
||||
inv_perm[1:] = torch.argsort(perm) + 1
|
||||
kv_cache[:, 1:blocks_end, ...] = kv_cache[:, perm, ...]
|
||||
kv_cache[1:blocks_end, ...] = kv_cache[perm, ...]
|
||||
|
||||
# Construct the right block table
|
||||
# Start from block_id=1 since block_id=0 is considered the null block
|
||||
@@ -473,28 +477,35 @@ def _test_backend_correctness(
|
||||
# Note: flex_attention has known Triton kernel compatibility issues
|
||||
# with test infrastructures
|
||||
for backend_name in backend_to_test:
|
||||
# FlashAttentionm + FlexAttention:
|
||||
# [2, num_blocks, block_size, num_kv_heads, head_size]
|
||||
# FlashInfer + Triton:
|
||||
# [num_blocks, 2, block_size, num_kv_heads, head_size]
|
||||
# Select the appropriate KV cache format for each backend
|
||||
kv_cache_for_backend = kv_cache
|
||||
reset_kv_cache_layout = False
|
||||
if backend_name in (
|
||||
AttentionBackendEnum.FLASHINFER,
|
||||
AttentionBackendEnum.TRITON_ATTN,
|
||||
):
|
||||
kv_cache_for_backend = kv_cache.transpose(0, 1)
|
||||
|
||||
# Resolve backend class for both enum and string names.
|
||||
actual_backend = backend_name
|
||||
if backend_name == "FLEX_ATTENTION_SLOW":
|
||||
actual_backend = AttentionBackendEnum.FLEX_ATTENTION
|
||||
if hasattr(actual_backend, "get_class"):
|
||||
backend_cls = actual_backend.get_class()
|
||||
else:
|
||||
backend_cls = None
|
||||
|
||||
if backend_name == AttentionBackendEnum.FLASHINFER:
|
||||
# For FlashInfer default to HND layout and
|
||||
kv_cache_for_backend = (
|
||||
kv_cache_for_backend.transpose(2, 3).contiguous().transpose(2, 3)
|
||||
)
|
||||
set_kv_cache_layout("HND")
|
||||
reset_kv_cache_layout = True
|
||||
elif backend_name == AttentionBackendEnum.TRITON_ATTN:
|
||||
kv_cache_for_backend = kv_cache_for_backend.contiguous()
|
||||
|
||||
# Apply stride order like runtime does in
|
||||
# _reshape_kv_cache (attn_utils.py:182-210): permute to physical
|
||||
# layout, make contiguous, then permute to logical layout.
|
||||
kv_cache_for_backend = kv_cache
|
||||
if backend_cls is not None:
|
||||
try:
|
||||
stride_order = backend_cls.get_kv_cache_stride_order()
|
||||
except (AttributeError, NotImplementedError):
|
||||
stride_order = tuple(range(kv_cache.ndim))
|
||||
if stride_order != tuple(range(kv_cache.ndim)):
|
||||
inv_order = [stride_order.index(i) for i in range(len(stride_order))]
|
||||
kv_cache_for_backend = (
|
||||
kv_cache.permute(*stride_order).contiguous().permute(*inv_order)
|
||||
)
|
||||
|
||||
try:
|
||||
backend_output = run_attention_backend(
|
||||
|
||||
@@ -19,7 +19,7 @@ size-1 dimensions via torch.as_strided — zero-copy.
|
||||
|
||||
The degenerate stride manifests at different positions in different backends:
|
||||
- FlashInfer: stride(-3) after kv_cache.permute() → shape [..., 1, B, D]
|
||||
- FlashAttention: stride(-2) after kv_cache.unbind(0) → shape [N, B, 1, D]
|
||||
- FlashAttention: stride(-2) after kv_cache.unbind(1) → shape [N, B, 1, D]
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
@@ -1,14 +1,13 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections import defaultdict
|
||||
from unittest.mock import MagicMock, patch
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import get_dtype_size
|
||||
from vllm.v1.attention.backend import AttentionBackend
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.backends.utils import set_kv_cache_layout
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
@@ -85,15 +84,6 @@ def _allocate_and_reshape_kv_caches(
|
||||
set_kv_cache_layout(None)
|
||||
|
||||
|
||||
def _make_mock_layer(backend_cls: type[AttentionBackend]):
|
||||
"""
|
||||
Create a mock AttentionLayerBase whose get_attn_backend returns backend_cls.
|
||||
"""
|
||||
layer = MagicMock()
|
||||
layer.get_attn_backend.return_value = backend_cls
|
||||
return layer
|
||||
|
||||
|
||||
def _make_worker(kv_cache_config: KVCacheConfig):
|
||||
"""
|
||||
Create an OffloadingConnectorWorker with mocked dependencies.
|
||||
@@ -119,11 +109,7 @@ def _make_worker(kv_cache_config: KVCacheConfig):
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", ATTN_BACKENDS)
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.offloading"
|
||||
".worker.get_layers_from_vllm_config"
|
||||
)
|
||||
def test_register_kv_caches(mock_get_layers, backend):
|
||||
def test_register_kv_caches(backend):
|
||||
"""Test register_kv_caches with multiple groups covering all layer types.
|
||||
|
||||
Creates one FullAttention group, one MLA group, one Mamba group, and
|
||||
@@ -287,13 +273,6 @@ def test_register_kv_caches(mock_get_layers, backend):
|
||||
device=torch.device("cuda:0"),
|
||||
)
|
||||
|
||||
mock_layers: dict[str, MagicMock] = {}
|
||||
for layer_name in attn_layer_names:
|
||||
mock_layers[layer_name] = _make_mock_layer(backend_cls)
|
||||
for layer_name in mla_layer_names:
|
||||
mock_layers[layer_name] = _make_mock_layer(DeepseekV32IndexerBackend)
|
||||
mock_get_layers.return_value = mock_layers
|
||||
|
||||
worker, spec = _make_worker(kv_cache_config)
|
||||
worker.register_kv_caches(kv_caches)
|
||||
|
||||
@@ -360,11 +339,7 @@ def test_register_kv_caches(mock_get_layers, backend):
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", ATTN_BACKENDS)
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.offloading"
|
||||
".worker.get_layers_from_vllm_config"
|
||||
)
|
||||
def test_register_kv_caches_uniform_type(mock_get_layers, backend):
|
||||
def test_register_kv_caches_uniform_type(backend):
|
||||
"""Test register_kv_caches with UniformTypeKVCacheSpecs.
|
||||
|
||||
Two attention layers use the same backend but different num_kv_heads,
|
||||
@@ -441,64 +416,29 @@ def test_register_kv_caches_uniform_type(mock_get_layers, backend):
|
||||
device=torch.device("cuda:0"),
|
||||
)
|
||||
|
||||
mock_get_layers.return_value = {
|
||||
layer_a: _make_mock_layer(backend_cls),
|
||||
layer_b: _make_mock_layer(backend_cls),
|
||||
}
|
||||
|
||||
worker, spec = _make_worker(kv_cache_config)
|
||||
worker.register_kv_caches(kv_caches)
|
||||
|
||||
canonical = spec.get_handlers.call_args[0][0]
|
||||
assert isinstance(canonical, CanonicalKVCaches)
|
||||
|
||||
unbinds = backend_cls.get_name() in ("FLASH_ATTN", "FLEX_ATTENTION")
|
||||
tensors_per_layer = 2 if unbinds else 1
|
||||
|
||||
for block_tensor in canonical.tensors:
|
||||
assert block_tensor.tensor.dtype == torch.int8
|
||||
|
||||
# Single group with refs from both layers
|
||||
assert len(canonical.group_data_refs) == 1
|
||||
group_refs = canonical.group_data_refs[0]
|
||||
assert len(group_refs) == 2 * tensors_per_layer
|
||||
assert len(group_refs) == 2
|
||||
|
||||
if unbinds:
|
||||
half_a = spec_a.page_size_bytes // 2
|
||||
half_b = spec_b.page_size_bytes // 2
|
||||
assert len(canonical.tensors) == 2
|
||||
assert canonical.tensors[0].page_size_bytes == spec_a.page_size_bytes
|
||||
assert canonical.tensors[1].page_size_bytes == spec_b.page_size_bytes
|
||||
assert canonical.tensors[0].tensor.shape == (NUM_BLOCKS, spec_a.page_size_bytes)
|
||||
assert canonical.tensors[1].tensor.shape == (NUM_BLOCKS, spec_b.page_size_bytes)
|
||||
|
||||
assert len(canonical.tensors) == 4
|
||||
assert canonical.tensors[0].page_size_bytes == half_a
|
||||
assert canonical.tensors[1].page_size_bytes == half_a
|
||||
assert canonical.tensors[2].page_size_bytes == half_b
|
||||
assert canonical.tensors[3].page_size_bytes == half_b
|
||||
assert canonical.tensors[0].tensor.shape == (NUM_BLOCKS, half_a)
|
||||
assert canonical.tensors[1].tensor.shape == (NUM_BLOCKS, half_a)
|
||||
assert canonical.tensors[2].tensor.shape == (NUM_BLOCKS, half_b)
|
||||
assert canonical.tensors[3].tensor.shape == (NUM_BLOCKS, half_b)
|
||||
|
||||
assert group_refs[0] == CanonicalKVCacheRef(
|
||||
tensor_idx=0, page_size_bytes=half_a
|
||||
)
|
||||
assert group_refs[1] == CanonicalKVCacheRef(
|
||||
tensor_idx=1, page_size_bytes=half_a
|
||||
)
|
||||
assert group_refs[2] == CanonicalKVCacheRef(
|
||||
tensor_idx=2, page_size_bytes=half_b
|
||||
)
|
||||
assert group_refs[3] == CanonicalKVCacheRef(
|
||||
tensor_idx=3, page_size_bytes=half_b
|
||||
)
|
||||
else:
|
||||
assert len(canonical.tensors) == 2
|
||||
assert canonical.tensors[0].page_size_bytes == spec_a.page_size_bytes
|
||||
assert canonical.tensors[1].page_size_bytes == spec_b.page_size_bytes
|
||||
assert canonical.tensors[0].tensor.shape == (NUM_BLOCKS, spec_a.page_size_bytes)
|
||||
assert canonical.tensors[1].tensor.shape == (NUM_BLOCKS, spec_b.page_size_bytes)
|
||||
|
||||
assert group_refs[0] == CanonicalKVCacheRef(
|
||||
tensor_idx=0, page_size_bytes=spec_a.page_size_bytes
|
||||
)
|
||||
assert group_refs[1] == CanonicalKVCacheRef(
|
||||
tensor_idx=1, page_size_bytes=spec_b.page_size_bytes
|
||||
)
|
||||
assert group_refs[0] == CanonicalKVCacheRef(
|
||||
tensor_idx=0, page_size_bytes=spec_a.page_size_bytes
|
||||
)
|
||||
assert group_refs[1] == CanonicalKVCacheRef(
|
||||
tensor_idx=1, page_size_bytes=spec_b.page_size_bytes
|
||||
)
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
from collections.abc import Iterable, Iterator
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -25,7 +25,6 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading_connector import (
|
||||
)
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.utils.hashing import sha256
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
get_request_block_hasher,
|
||||
init_none_hash,
|
||||
@@ -239,37 +238,22 @@ class RequestRunner:
|
||||
|
||||
# register worker kv_caches to enable OffloadingWorker creations
|
||||
# set_current_vllm_config is needed for get_kv_cache_layout() to work
|
||||
# Mock get_layers_from_vllm_config so that mock layer names
|
||||
# resolve to layers whose get_attn_backend() returns
|
||||
# FlashAttentionBackend.
|
||||
def _mock_get_layers(_vllm_config, _layer_type, layer_names):
|
||||
mock_layer = MagicMock()
|
||||
mock_layer.get_attn_backend.return_value = FlashAttentionBackend
|
||||
return {name: mock_layer for name in layer_names}
|
||||
|
||||
kv_caches: dict[str, torch.Tensor] = {}
|
||||
for group in kv_cache_groups:
|
||||
spec = group.kv_cache_spec
|
||||
for layer_name in group.layer_names:
|
||||
# Shape follows FlashAttention layout:
|
||||
# (2, num_blocks, block_size, num_kv_heads, head_size)
|
||||
# Shape: (num_blocks, 2, block_size, num_kv_heads, head_size)
|
||||
kv_caches[layer_name] = torch.empty(
|
||||
2,
|
||||
num_gpu_blocks,
|
||||
2,
|
||||
spec.block_size,
|
||||
spec.num_kv_heads,
|
||||
spec.head_size,
|
||||
dtype=spec.dtype,
|
||||
)
|
||||
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1"
|
||||
".offloading.worker.get_layers_from_vllm_config",
|
||||
side_effect=_mock_get_layers,
|
||||
),
|
||||
):
|
||||
with set_current_vllm_config(vllm_config):
|
||||
self.worker_connector.register_kv_caches(kv_caches)
|
||||
|
||||
# extract connector of scheduler
|
||||
|
||||
@@ -369,15 +369,31 @@ async def test_kv_producer(monkeypatch):
|
||||
with patch.object(
|
||||
prefill_worker, "_send_blocks", return_value=0
|
||||
) as mock_send_blocks:
|
||||
# With blocks-first layout, each block is virtually split
|
||||
# into K and V halves, producing non-coalesced transfers.
|
||||
kv_half = block_len // 2
|
||||
|
||||
def expected_split_transfers(src_base, dst_base, src_blocks, dst_blocks):
|
||||
"""Build expected (src_ptrs, dst_ptrs, lengths) for
|
||||
virtual-split K/V transfers."""
|
||||
src_ptrs, dst_ptrs, lengths = [], [], []
|
||||
for kv_offset in (0, kv_half):
|
||||
for sb, db in zip(src_blocks, dst_blocks):
|
||||
src_ptrs.append(src_base + sb * block_len + kv_offset)
|
||||
dst_ptrs.append(dst_base + db * block_len + kv_offset)
|
||||
lengths.append(kv_half)
|
||||
return src_ptrs, dst_ptrs, lengths
|
||||
|
||||
# Normal case: 2 blocks to 2 blocks
|
||||
# Worker processes the consumer's request
|
||||
await prefill_worker.send_kv_to_decode(identity, mock_socket, xfer_meta)
|
||||
# Verify transfer parameters are correct
|
||||
src_ptr = 0x1000 + 10 * block_len
|
||||
dst_ptr = 0x2000 + 20 * block_len
|
||||
length = 2 * block_len
|
||||
src, dst, lens = expected_split_transfers(
|
||||
0x1000, 0x2000, [10, 11], [20, 21]
|
||||
)
|
||||
mock_send_blocks.assert_called_once_with(
|
||||
"consumer-host:54321", [src_ptr], [dst_ptr], [length]
|
||||
"consumer-host:54321",
|
||||
src,
|
||||
dst,
|
||||
lens,
|
||||
)
|
||||
mock_socket.send_multipart.assert_called_once()
|
||||
|
||||
@@ -404,11 +420,12 @@ async def test_kv_producer(monkeypatch):
|
||||
# Worker processes the consumer's request
|
||||
await prefill_worker.send_kv_to_decode(identity, mock_socket, xfer_meta)
|
||||
# Verify transfer parameters are correct: 11 to 20
|
||||
src_ptr = 0x1000 + 11 * block_len
|
||||
dst_ptr = 0x2000 + 20 * block_len
|
||||
length = 1 * block_len
|
||||
src, dst, lens = expected_split_transfers(0x1000, 0x2000, [11], [20])
|
||||
mock_send_blocks.assert_called_once_with(
|
||||
"consumer-host:54321", [src_ptr], [dst_ptr], [length]
|
||||
"consumer-host:54321",
|
||||
src,
|
||||
dst,
|
||||
lens,
|
||||
)
|
||||
mock_socket.send_multipart.assert_called_once()
|
||||
|
||||
@@ -618,18 +635,14 @@ def test_register_kv_caches():
|
||||
|
||||
mock_batch_register.assert_called_once()
|
||||
registered_ptrs, registered_lens = mock_batch_register.call_args[0]
|
||||
expected_ptrs = {
|
||||
tensor.data_ptr()
|
||||
for kv_pair in kv_caches.values()
|
||||
for tensor in kv_pair
|
||||
}
|
||||
expected_ptrs = {tensor.data_ptr() for tensor in kv_caches.values()}
|
||||
assert set(registered_ptrs) == expected_ptrs
|
||||
assert set(registered_lens) == {tensor1[0].nbytes}
|
||||
assert set(registered_lens) == {tensor1.nbytes}
|
||||
|
||||
# Verify block_len_per_layer is set correctly.
|
||||
assert len(worker.block_len_per_layer) == len(registered_ptrs)
|
||||
for bl in worker.block_len_per_layer:
|
||||
assert bl == tensor1[0].nbytes // tensor1.shape[1]
|
||||
assert bl == tensor1.nbytes // tensor1.shape[0]
|
||||
|
||||
|
||||
def test_register_kv_caches_supports_mixed_mla_and_eagle_shapes():
|
||||
@@ -791,33 +804,49 @@ async def test_kv_producer_heterogeneous_tp(monkeypatch, d_tp_size):
|
||||
# Flatten nested per-group block IDs for assertions
|
||||
flat_local = [b for g in local_block_ids for b in g]
|
||||
flat_remote = [b for g in remote_block_ids for b in g]
|
||||
num_blocks = len(flat_local)
|
||||
|
||||
# Heterogeneous TP: blocks cannot be coalesced because
|
||||
# local and remote block_lens differ
|
||||
assert len(src_ptrs) == len(flat_local)
|
||||
assert len(dst_ptrs) == len(flat_local)
|
||||
assert len(lengths) == len(flat_local)
|
||||
# With blocks-first layout, virtual split halves block
|
||||
# lengths and doubles transfer regions (K + V).
|
||||
local_kv_block_len = local_block_len // 2
|
||||
remote_kv_block_len = remote_block_len // 2
|
||||
|
||||
# Compute expected offsets based on TP ratio
|
||||
assert len(src_ptrs) == 2 * num_blocks
|
||||
assert len(dst_ptrs) == 2 * num_blocks
|
||||
assert len(lengths) == 2 * num_blocks
|
||||
|
||||
# Compute expected offsets using kv_block_len
|
||||
if d_tp_size <= P_TP_SIZE:
|
||||
tp_ratio = P_TP_SIZE // d_tp_size
|
||||
expected_src_off = 0
|
||||
expected_dst_off = (P_TP_RANK % tp_ratio) * local_block_len
|
||||
expected_xfer_len = local_block_len
|
||||
expected_dst_off = (P_TP_RANK % tp_ratio) * local_kv_block_len
|
||||
expected_xfer_len = local_kv_block_len
|
||||
else:
|
||||
ratio_abs = d_tp_size // P_TP_SIZE
|
||||
expected_src_off = (d_rank % ratio_abs) * remote_block_len
|
||||
expected_src_off = (d_rank % ratio_abs) * remote_kv_block_len
|
||||
expected_dst_off = 0
|
||||
expected_xfer_len = remote_block_len
|
||||
expected_xfer_len = remote_kv_block_len
|
||||
|
||||
for idx, (lblk, rblk) in enumerate(zip(flat_local, flat_remote)):
|
||||
assert src_ptrs[idx] == (
|
||||
0x1000 + lblk * local_block_len + expected_src_off
|
||||
)
|
||||
assert dst_ptrs[idx] == (
|
||||
0x2000 + rblk * remote_block_len + expected_dst_off
|
||||
)
|
||||
assert lengths[idx] == expected_xfer_len
|
||||
# First num_blocks entries are K region,
|
||||
# next num_blocks are V region.
|
||||
for region_idx in range(2):
|
||||
local_region_base = 0x1000 + region_idx * local_kv_block_len
|
||||
remote_region_base = 0x2000 + region_idx * remote_kv_block_len
|
||||
for blk_idx, (lblk, rblk) in enumerate(
|
||||
zip(flat_local, flat_remote)
|
||||
):
|
||||
idx = region_idx * num_blocks + blk_idx
|
||||
assert src_ptrs[idx] == (
|
||||
local_region_base
|
||||
+ lblk * local_block_len
|
||||
+ expected_src_off
|
||||
)
|
||||
assert dst_ptrs[idx] == (
|
||||
remote_region_base
|
||||
+ rblk * remote_block_len
|
||||
+ expected_dst_off
|
||||
)
|
||||
assert lengths[idx] == expected_xfer_len
|
||||
|
||||
# Verify successful response sent back to consumer
|
||||
mock_socket.send_multipart.assert_called_once()
|
||||
|
||||
@@ -1635,6 +1635,7 @@ def test_register_kv_caches(
|
||||
num_blocks=1, block_size=16, num_kv_heads=1, head_size=1
|
||||
)
|
||||
is_blocks_first = len(test_shape) == 5 and test_shape[0] == 1
|
||||
virtually_split = is_blocks_first and not connector.prefer_cross_layer_blocks
|
||||
|
||||
if connector.prefer_cross_layer_blocks:
|
||||
with set_current_vllm_config(vllm_config):
|
||||
@@ -1665,7 +1666,7 @@ def test_register_kv_caches(
|
||||
]
|
||||
expected_num_entries = 1
|
||||
|
||||
expected_blocks_count = num_blocks * (2 if is_blocks_first else 1)
|
||||
expected_blocks_count = num_blocks * (2 if virtually_split else 1)
|
||||
|
||||
kv_caches = {"all-layers": cross_layers_kv_cache}
|
||||
else:
|
||||
@@ -1739,7 +1740,7 @@ def test_register_kv_caches(
|
||||
else:
|
||||
num_blocks = kv_cache_config.num_blocks
|
||||
|
||||
if is_blocks_first:
|
||||
if virtually_split:
|
||||
expected_block_len = expected_tensor_size // num_blocks // 2
|
||||
else:
|
||||
expected_block_len = expected_tensor_size // num_blocks
|
||||
|
||||
@@ -34,7 +34,6 @@ from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.core.kv_cache_utils import estimate_max_model_len, get_kv_cache_configs
|
||||
from vllm.v1.core.sched.output import CachedRequestData, NewRequestData, SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
@@ -44,7 +43,7 @@ from vllm.v1.sample.metadata import SamplingMetadata
|
||||
from vllm.v1.spec_decode.metadata import SpecDecodeMetadata
|
||||
from vllm.v1.worker.gpu_input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
from vllm.v1.worker.utils import AttentionGroup, select_common_block_size
|
||||
from vllm.v1.worker.utils import select_common_block_size
|
||||
|
||||
BLOCK_SIZE = 16
|
||||
NUM_BLOCKS = 10
|
||||
@@ -1195,33 +1194,6 @@ def test_hybrid_attention_mamba_tensor_shapes():
|
||||
assert torch.equal(actual_ssm, expected_ssm)
|
||||
|
||||
|
||||
def test_update_hybrid_attention_mamba_layout_with_num_block_2_rewrites_stride():
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||
|
||||
ambiguous_cache = torch.empty((2, 2, BLOCK_SIZE, 1, 8), dtype=torch.float16)
|
||||
"""Ambiguous, because both dims[0=kv_dim] and dims[1=num_blocks] == 2"""
|
||||
hidden_size = ambiguous_cache.shape[2:].numel()
|
||||
assert ambiguous_cache.stride()[:2] == (2 * hidden_size, hidden_size)
|
||||
|
||||
attention_spec = AttentionSpec(
|
||||
block_size=BLOCK_SIZE, num_kv_heads=1, head_size=8, dtype=torch.float16
|
||||
)
|
||||
runner_stub = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(cache_dtype="auto"),
|
||||
_kv_cache_spec_attn_group_iterator=lambda: iter(
|
||||
[AttentionGroup(FlashAttentionBackend, ["attn"], attention_spec, 0)]
|
||||
),
|
||||
)
|
||||
GPUModelRunner._update_hybrid_attention_mamba_layout(
|
||||
runner_stub, {"attn": ambiguous_cache}, [BLOCK_SIZE]
|
||||
)
|
||||
|
||||
assert ambiguous_cache.stride()[:2] == (hidden_size, 2 * hidden_size), """\
|
||||
We expect _update_hybrid_attention_mamba_layout to re-stride the cache from:
|
||||
(2, num_blocks) -> (num_blocks, 2), even when num_blocks==2,
|
||||
which was ambiguous before get_kv_cache_block_dim was used"""
|
||||
|
||||
|
||||
def test_hybrid_block_table_initialization():
|
||||
"""Test hybrid block table with different kernel and kvcache_manager block
|
||||
sizes."""
|
||||
|
||||
Reference in New Issue
Block a user