[6/N][KV-Cache Layout Refactor] Standardize KV cache layout (#51718)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
@@ -1,380 +1,237 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for contiguous KV cache packing."""
|
||||
"""Tests for contiguous KV cache packing.
|
||||
|
||||
Every cache group packs its layers densely into one block; groups overlay each other
|
||||
(a block ID is owned by one group at a time), so the packed block stride is the
|
||||
largest group's packing. The layout decides whether the layer dim sits outside the
|
||||
block dim (a contiguous region per layer) or inside it (all layers' pages within each
|
||||
block); the allocation is the same either way.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import CacheConfig
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
_get_kv_cache_config_packed,
|
||||
_get_kv_cache_groups_uniform_groups,
|
||||
_get_kv_cache_bytes_per_block,
|
||||
_pool_bytes_per_block,
|
||||
get_kv_cache_config_from_groups,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
KVCacheLayout,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
SlidingWindowSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
from vllm.v1.worker.utils import allocate_kv_cache
|
||||
|
||||
MEMORY = 8 * 1024 * 1024
|
||||
|
||||
|
||||
def _make_mla_spec(page_size: int, block_size: int = 256) -> MLAAttentionSpec:
|
||||
def _mla(head_size: int) -> MLAAttentionSpec:
|
||||
return MLAAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=512,
|
||||
dtype=torch.uint8,
|
||||
page_size_padded=page_size,
|
||||
cache_dtype_str="fp8_ds_mla",
|
||||
model_version="deepseek_v4",
|
||||
alignment=576,
|
||||
block_size=64, num_kv_heads=1, head_size=head_size, dtype=torch.uint8
|
||||
)
|
||||
|
||||
|
||||
def _make_full_spec() -> FullAttentionSpec:
|
||||
def _full() -> FullAttentionSpec:
|
||||
return FullAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=2,
|
||||
head_size=64,
|
||||
dtype=torch.float16,
|
||||
block_size=16, num_kv_heads=2, head_size=64, dtype=torch.float16
|
||||
)
|
||||
|
||||
|
||||
def _make_sw_spec() -> SlidingWindowSpec:
|
||||
return SlidingWindowSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=2,
|
||||
head_size=64,
|
||||
dtype=torch.float16,
|
||||
sliding_window=128,
|
||||
def _uniform_group(specs: dict) -> KVCacheGroupSpec:
|
||||
return KVCacheGroupSpec(
|
||||
list(specs),
|
||||
UniformTypeKVCacheSpecs(block_size=64, kv_cache_specs=specs),
|
||||
)
|
||||
|
||||
|
||||
def _make_groups(n_c4, n_c128, n_swa):
|
||||
PS_C4_MLA = 37440
|
||||
PS_C4_IDX = 8640
|
||||
PS_C128 = 1728
|
||||
PS_SWA = 37440
|
||||
|
||||
mla_specs = {}
|
||||
for i in range(n_c4):
|
||||
mla_specs[f"c4_mla.{i}"] = _make_mla_spec(PS_C4_MLA)
|
||||
mla_specs[f"c4_idx.{i}"] = _make_mla_spec(PS_C4_IDX)
|
||||
for i in range(n_c128):
|
||||
mla_specs[f"c128_mla.{i}"] = _make_mla_spec(PS_C128)
|
||||
|
||||
mla_group = KVCacheGroupSpec(
|
||||
layer_names=list(mla_specs.keys()),
|
||||
kv_cache_spec=UniformTypeKVCacheSpecs(block_size=256, kv_cache_specs=mla_specs),
|
||||
)
|
||||
|
||||
swa_specs = {}
|
||||
for i in range(n_swa):
|
||||
swa_specs[f"swa.{i}"] = _make_mla_spec(PS_SWA)
|
||||
|
||||
swa_group = KVCacheGroupSpec(
|
||||
layer_names=list(swa_specs.keys()),
|
||||
kv_cache_spec=UniformTypeKVCacheSpecs(block_size=256, kv_cache_specs=swa_specs),
|
||||
)
|
||||
|
||||
return [mla_group, swa_group]
|
||||
def _mixed_page_groups(n_mla=3, n_idx=3, n_swa=5):
|
||||
"""A DeepSeek-V4-style hybrid: two groups with different page mixes."""
|
||||
g1 = {f"mla.{i}": _mla(512) for i in range(n_mla)}
|
||||
g1.update({f"idx.{i}": _mla(128) for i in range(n_idx)})
|
||||
g2 = {f"swa.{i}": _mla(512) for i in range(n_swa)}
|
||||
return [_uniform_group(g1), _uniform_group(g2)], g1, g2
|
||||
|
||||
|
||||
def _mock_vllm_config(kv_connector_extra_config: dict[str, str] | None = None):
|
||||
def _mock_vllm_config(layout: str | None):
|
||||
config = MagicMock()
|
||||
config.cache_config = CacheConfig()
|
||||
config.cache_config.num_gpu_blocks_override = None
|
||||
config.cache_config.prefix_cache_retention_interval = 0
|
||||
config.kv_transfer_config = None
|
||||
if kv_connector_extra_config is not None:
|
||||
config.kv_transfer_config = MagicMock()
|
||||
config.kv_transfer_config.kv_connector_extra_config = kv_connector_extra_config
|
||||
config.cache_config.kv_cache_layout = layout
|
||||
return config
|
||||
|
||||
|
||||
def _run(n_c4=3, n_c128=2, n_swa=5, mem=100 * 1024 * 1024):
|
||||
groups = _make_groups(n_c4, n_c128, n_swa)
|
||||
return _get_kv_cache_config_packed(_mock_vllm_config(), groups, mem)
|
||||
|
||||
|
||||
def _page_sizes_by_layer(
|
||||
groups: list[KVCacheGroupSpec],
|
||||
) -> dict[str, int]:
|
||||
page_sizes = {}
|
||||
for group in groups:
|
||||
specs = group.kv_cache_spec.kv_cache_specs
|
||||
for layer_name in group.layer_names:
|
||||
page_sizes[layer_name] = specs[layer_name].page_size_bytes
|
||||
return page_sizes
|
||||
|
||||
|
||||
def _packing_by_layer(
|
||||
tensors: list[KVCacheTensor],
|
||||
) -> dict[str, tuple[int, int]]:
|
||||
def _pages(groups) -> dict[str, int]:
|
||||
return {
|
||||
layer_name: (tensor.offset, tensor.block_stride)
|
||||
for tensor in tensors
|
||||
for layer_name in tensor.shared_by
|
||||
name: group.kv_cache_spec.kv_cache_specs[name].page_size_bytes
|
||||
if isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
|
||||
else group.kv_cache_spec.page_size_bytes
|
||||
for group in groups
|
||||
for name in group.layer_names
|
||||
}
|
||||
|
||||
|
||||
def _make_views(
|
||||
groups: list[KVCacheGroupSpec],
|
||||
num_blocks: int,
|
||||
tensors: list[KVCacheTensor],
|
||||
) -> dict[str, torch.Tensor]:
|
||||
page_sizes = _page_sizes_by_layer(groups)
|
||||
packing = _packing_by_layer(tensors)
|
||||
backing = torch.zeros(tensors[0].size, dtype=torch.uint8)
|
||||
return {
|
||||
layer_name: torch.as_strided(
|
||||
backing,
|
||||
size=(num_blocks, page_size),
|
||||
stride=(packing[layer_name][1], 1),
|
||||
storage_offset=packing[layer_name][0],
|
||||
def _expected_bytes_per_block(groups) -> int:
|
||||
pages = _pages(groups)
|
||||
return max(sum(pages[n] for n in g.layer_names) for g in groups)
|
||||
|
||||
|
||||
def _bind(config, layout: str):
|
||||
return allocate_kv_cache(config, torch.device("cpu"), KVCacheLayout[layout], None)
|
||||
|
||||
|
||||
class TestDensePacking:
|
||||
def test_bytes_per_block_is_largest_group(self):
|
||||
groups, g1, g2 = _mixed_page_groups()
|
||||
assert _get_kv_cache_bytes_per_block(groups) == _expected_bytes_per_block(
|
||||
groups
|
||||
)
|
||||
for layer_name, page_size in page_sizes.items()
|
||||
}
|
||||
|
||||
|
||||
def _make_page_group(prefix: str, page_sizes: list[int]) -> KVCacheGroupSpec:
|
||||
specs = {
|
||||
f"{prefix}.{i}": MagicMock(page_size_bytes=page_size)
|
||||
for i, page_size in enumerate(page_sizes)
|
||||
}
|
||||
return KVCacheGroupSpec(
|
||||
layer_names=list(specs),
|
||||
kv_cache_spec=UniformTypeKVCacheSpecs(block_size=256, kv_cache_specs=specs),
|
||||
)
|
||||
|
||||
|
||||
class TestInterleavedPacking:
|
||||
def test_compact_cache_overlays_fp32_state_group(self):
|
||||
full_specs = {}
|
||||
state_specs = {}
|
||||
for i in range(2):
|
||||
full_specs[f"mla.{i}"] = MLAAttentionSpec(
|
||||
block_size=256,
|
||||
num_kv_heads=1,
|
||||
head_size=512,
|
||||
dtype=torch.uint8,
|
||||
page_size_padded=32768,
|
||||
indexes_kv_by_block_stride=True,
|
||||
compress_ratio=4,
|
||||
)
|
||||
full_specs[f"indexer.{i}"] = MLAAttentionSpec(
|
||||
block_size=256,
|
||||
num_kv_heads=1,
|
||||
head_size=68,
|
||||
dtype=torch.uint8,
|
||||
page_size_padded=4608,
|
||||
compress_ratio=4,
|
||||
)
|
||||
state_specs[f"mla_state.{i}"] = SlidingWindowMLASpec(
|
||||
block_size=4,
|
||||
num_kv_heads=1,
|
||||
head_size=2048,
|
||||
dtype=torch.float32,
|
||||
sliding_window=8,
|
||||
indexes_kv_by_block_stride=True,
|
||||
)
|
||||
state_specs[f"indexer_state.{i}"] = SlidingWindowMLASpec(
|
||||
block_size=4,
|
||||
num_kv_heads=1,
|
||||
head_size=512,
|
||||
dtype=torch.float32,
|
||||
sliding_window=8,
|
||||
indexes_kv_by_block_stride=True,
|
||||
)
|
||||
|
||||
grouped_specs = [
|
||||
UniformTypeKVCacheSpecs(block_size=256, kv_cache_specs=full_specs),
|
||||
UniformTypeKVCacheSpecs(block_size=4, kv_cache_specs=state_specs),
|
||||
config = get_kv_cache_config_from_groups(
|
||||
_mock_vllm_config("BLHNC"), groups, MEMORY
|
||||
)
|
||||
# Groups overlay: both start at offset 0.
|
||||
assert [tensor.offset for tensor in config.kv_cache_tensors].count(0) == 2
|
||||
assert [tensor.layers for tensor in config.kv_cache_tensors] == [
|
||||
list(g1)[:3],
|
||||
list(g1)[3:],
|
||||
list(g2),
|
||||
]
|
||||
groups = _get_kv_cache_groups_uniform_groups(grouped_specs)
|
||||
|
||||
assert len(groups) == 2
|
||||
assert {full_specs[f"indexer.{i}"].page_size_bytes for i in range(2)} == {4608}
|
||||
assert {full_specs[f"indexer.{i}"].real_page_size_bytes for i in range(2)} == {
|
||||
4352
|
||||
def test_layers_within_a_group_are_dense(self):
|
||||
groups, _, _ = _mixed_page_groups()
|
||||
pages = _pages(groups)
|
||||
config = get_kv_cache_config_from_groups(
|
||||
_mock_vllm_config("BLHNC"), groups, MEMORY
|
||||
)
|
||||
offsets = {
|
||||
name: tensor.offset + i * tensor.layer_stride
|
||||
for tensor in config.kv_cache_tensors
|
||||
for i, name in enumerate(tensor.layers)
|
||||
}
|
||||
assert {
|
||||
state_specs[f"indexer_state.{i}"].page_size_bytes for i in range(2)
|
||||
} == {8192}
|
||||
|
||||
full_group_bytes = 2 * (32768 + 4608)
|
||||
state_group_bytes = 2 * (32768 + 8192)
|
||||
bytes_per_block = max(full_group_bytes, state_group_bytes)
|
||||
num_blocks, tensors = _get_kv_cache_config_packed(
|
||||
_mock_vllm_config(), groups, bytes_per_block * 32
|
||||
)
|
||||
assert num_blocks == 32
|
||||
assert {tensor.block_stride for tensor in tensors} == {bytes_per_block}
|
||||
|
||||
packing = _packing_by_layer(tensors)
|
||||
assert packing["mla.0"][0] == packing["mla_state.0"][0] == 0
|
||||
assert packing["indexer.0"][0] == 32768
|
||||
assert packing["indexer_state.0"][0] == 32768
|
||||
|
||||
def test_deepseek_v4_pro_stride(self):
|
||||
groups = [
|
||||
_make_page_group("full", [32768, 4608] * 30 + [1024] * 31),
|
||||
_make_page_group("c4_state", [32768, 8192] * 30),
|
||||
_make_page_group("c128_state", [32768] * 31),
|
||||
_make_page_group("swa.0", [32768] * 31),
|
||||
_make_page_group("swa.1", [32768] * 30),
|
||||
]
|
||||
expected_stride = 1_228_800
|
||||
|
||||
num_blocks, tensors = _get_kv_cache_config_packed(
|
||||
_mock_vllm_config(), groups, expected_stride * 32
|
||||
)
|
||||
|
||||
assert num_blocks == 32
|
||||
assert {tensor.block_stride for tensor in tensors} == {expected_stride}
|
||||
assert {tensor.size for tensor in tensors} == {expected_stride * 32}
|
||||
|
||||
def test_all_tensors_have_block_stride(self):
|
||||
_, tensors = _run()
|
||||
for t in tensors:
|
||||
assert t.block_stride > 0
|
||||
|
||||
def test_all_tensors_share_same_size(self):
|
||||
_, tensors = _run()
|
||||
sizes = set(t.size for t in tensors)
|
||||
assert len(sizes) == 1
|
||||
assert sizes.pop() > 0
|
||||
|
||||
def test_offsets_within_one_block(self):
|
||||
groups = _make_groups(n_c4=3, n_c128=2, n_swa=5)
|
||||
_, tensors = _get_kv_cache_config_packed(
|
||||
_mock_vllm_config(), groups, 100 * 1024 * 1024
|
||||
)
|
||||
page_sizes = _page_sizes_by_layer(groups)
|
||||
packing = _packing_by_layer(tensors)
|
||||
for layer_name, page_size in page_sizes.items():
|
||||
offset, block_stride = packing[layer_name]
|
||||
assert offset + page_size <= block_stride
|
||||
|
||||
def test_layouts_are_disjoint_within_each_group(self):
|
||||
groups = _make_groups(n_c4=3, n_c128=2, n_swa=5)
|
||||
_, tensors = _get_kv_cache_config_packed(
|
||||
_mock_vllm_config(), groups, 100 * 1024 * 1024
|
||||
)
|
||||
page_sizes = _page_sizes_by_layer(groups)
|
||||
packing = _packing_by_layer(tensors)
|
||||
|
||||
for group in groups:
|
||||
ranges = sorted(
|
||||
(packing[name][0], packing[name][0] + page_sizes[name])
|
||||
for name in group.layer_names
|
||||
expected = 0
|
||||
for name in group.layer_names:
|
||||
assert offsets[name] == expected
|
||||
expected += pages[name]
|
||||
|
||||
@pytest.mark.parametrize("layout", ["LBNHC", "BLHNC"])
|
||||
def test_allocation_is_layout_invariant(self, layout):
|
||||
specs = {f"l.{i}": _full() for i in range(4)}
|
||||
groups = [KVCacheGroupSpec(list(specs), _full())]
|
||||
config = get_kv_cache_config_from_groups(
|
||||
_mock_vllm_config(layout), groups, MEMORY
|
||||
)
|
||||
(tensor,) = config.kv_cache_tensors
|
||||
page = _full().page_size_bytes
|
||||
assert tensor.layers == list(specs)
|
||||
assert config.num_blocks == MEMORY // (4 * page)
|
||||
assert tensor.size == 4 * page * config.num_blocks
|
||||
if layout == "LBNHC":
|
||||
assert (tensor.layer_stride, tensor.block_stride) == (
|
||||
page * config.num_blocks,
|
||||
page,
|
||||
)
|
||||
assert all(left[1] <= right[0] for left, right in zip(ranges, ranges[1:]))
|
||||
|
||||
def test_all_layers_accounted_for(self):
|
||||
n_c4, n_c128, n_swa = 5, 4, 7
|
||||
_, tensors = _run(n_c4=n_c4, n_c128=n_c128, n_swa=n_swa)
|
||||
all_names = set()
|
||||
for t in tensors:
|
||||
all_names.update(t.shared_by)
|
||||
expected = n_c4 * 2 + n_c128 + n_swa
|
||||
assert len(all_names) == expected
|
||||
|
||||
def test_group_owned_blocks_do_not_alias(self):
|
||||
groups = _make_groups(n_c4=3, n_c128=2, n_swa=5)
|
||||
num_blocks, tensors = _get_kv_cache_config_packed(
|
||||
_mock_vllm_config(), groups, 100 * 1024 * 1024
|
||||
)
|
||||
views = _make_views(groups, num_blocks, tensors)
|
||||
|
||||
expected = {}
|
||||
value = 1
|
||||
for block_id, group in enumerate(groups):
|
||||
for layer_name in group.layer_names:
|
||||
views[layer_name][block_id].fill_(value)
|
||||
expected[layer_name] = (block_id, value)
|
||||
value += 1
|
||||
|
||||
for layer_name, (block_id, value) in expected.items():
|
||||
assert (views[layer_name][block_id] == value).all()
|
||||
|
||||
# Once the first group releases its block, another group may reuse it.
|
||||
for layer_name in groups[1].layer_names:
|
||||
views[layer_name][0].fill_(255)
|
||||
for layer_name in groups[1].layer_names:
|
||||
assert (views[layer_name][0] == 255).all()
|
||||
|
||||
def test_hma_attention_groups_keep_default_backing(self):
|
||||
full = _make_full_spec()
|
||||
sw = _make_sw_spec()
|
||||
page_size = full.page_size_bytes
|
||||
groups = [
|
||||
KVCacheGroupSpec(["full.0", "full.1"], full),
|
||||
KVCacheGroupSpec(["sw.0", "sw.2"], sw),
|
||||
KVCacheGroupSpec(["sw.1", "sw.3"], sw),
|
||||
]
|
||||
else:
|
||||
assert (tensor.layer_stride, tensor.block_stride) == (page, 4 * page)
|
||||
|
||||
@pytest.mark.parametrize("layout", ["LBNHC", "BLHNC"])
|
||||
def test_single_group_mixed_pages_follows_layout(self, layout):
|
||||
specs = {"mla.0": _mla(512), "mla.1": _mla(512), "idx.0": _mla(128)}
|
||||
groups = [_uniform_group(specs)]
|
||||
config = get_kv_cache_config_from_groups(
|
||||
_mock_vllm_config(), groups, available_memory=page_size * 2 * 32
|
||||
_mock_vllm_config(layout), groups, MEMORY
|
||||
)
|
||||
block_stride = _expected_bytes_per_block(groups)
|
||||
mla_tensor, idx_tensor = config.kv_cache_tensors
|
||||
assert mla_tensor.layers == ["mla.0", "mla.1"]
|
||||
assert idx_tensor.layers == ["idx.0"]
|
||||
assert {t.size for t in config.kv_cache_tensors} == {
|
||||
block_stride * config.num_blocks
|
||||
}
|
||||
if layout == "LBNHC":
|
||||
assert (
|
||||
idx_tensor.offset == 2 * _mla(512).page_size_bytes * config.num_blocks
|
||||
)
|
||||
assert mla_tensor.block_stride == _mla(512).page_size_bytes
|
||||
else:
|
||||
assert idx_tensor.offset == 2 * _mla(512).page_size_bytes
|
||||
assert mla_tensor.block_stride == block_stride
|
||||
|
||||
assert config.num_blocks == 32
|
||||
assert config.prefix_cache_retention_interval == 0
|
||||
assert sum(t.size for t in config.kv_cache_tensors) == page_size * 2 * 32
|
||||
assert config.kv_cache_tensors == [
|
||||
KVCacheTensor(size=page_size * 32, shared_by=["full.0", "sw.0", "sw.1"]),
|
||||
KVCacheTensor(size=page_size * 32, shared_by=["full.1", "sw.2", "sw.3"]),
|
||||
]
|
||||
|
||||
def test_hma_attention_groups_use_packed_backing_with_enable_cross_layers(self):
|
||||
full = _make_full_spec()
|
||||
sw = _make_sw_spec()
|
||||
page_size = full.page_size_bytes
|
||||
groups = [
|
||||
KVCacheGroupSpec(["full.0", "full.1"], full),
|
||||
KVCacheGroupSpec(["sw.0", "sw.2"], sw),
|
||||
KVCacheGroupSpec(["sw.1", "sw.3"], sw),
|
||||
]
|
||||
|
||||
def test_overlaid_groups_alias_and_stay_isolated(self):
|
||||
groups, g1, g2 = _mixed_page_groups()
|
||||
# Overlay models resolve to a block-outer layout at backend selection (the
|
||||
# model's backend declares it); mirror that here.
|
||||
config = get_kv_cache_config_from_groups(
|
||||
_mock_vllm_config({"enable_cross_layers_blocks": "True"}),
|
||||
groups,
|
||||
available_memory=page_size * 2 * 32,
|
||||
_mock_vllm_config("BLNHC"), groups, MEMORY
|
||||
)
|
||||
assert config.num_blocks == MEMORY // _expected_bytes_per_block(groups)
|
||||
assert _pool_bytes_per_block(groups) == _expected_bytes_per_block(groups)
|
||||
|
||||
assert config.num_blocks == 32
|
||||
assert {t.size for t in config.kv_cache_tensors} == {page_size * 2 * 32}
|
||||
assert config.kv_cache_tensors == [
|
||||
KVCacheTensor(
|
||||
size=page_size * 2 * 32,
|
||||
shared_by=["full.0", "sw.0", "sw.1"],
|
||||
offset=0,
|
||||
block_stride=page_size * 2,
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=page_size * 2 * 32,
|
||||
shared_by=["full.1", "sw.2", "sw.3"],
|
||||
offset=page_size,
|
||||
block_stride=page_size * 2,
|
||||
),
|
||||
]
|
||||
views = _bind(config, "BLNHC")
|
||||
assert set(views) == set(g1) | set(g2)
|
||||
assert views["swa.0"].data_ptr() == views["mla.0"].data_ptr()
|
||||
|
||||
def test_single_group_attention_keeps_unpacked_layout(self):
|
||||
spec = _make_full_spec()
|
||||
groups = [KVCacheGroupSpec(["full.0", "full.1"], spec)]
|
||||
# A block is owned by one group at a time: writes to group-owned
|
||||
# blocks never disturb the other group's blocks.
|
||||
for i, name in enumerate(g1):
|
||||
views[name][0].fill_(i + 1)
|
||||
views[name][2].fill_(i + 1)
|
||||
for i, name in enumerate(g2):
|
||||
views[name][1].fill_(100 + i)
|
||||
views[name][3].fill_(100 + i)
|
||||
for i, name in enumerate(g1):
|
||||
assert (views[name][0].to(torch.int32) == i + 1).all()
|
||||
assert (views[name][2].to(torch.int32) == i + 1).all()
|
||||
for i, name in enumerate(g2):
|
||||
assert (views[name][1].to(torch.int32) == 100 + i).all()
|
||||
assert (views[name][3].to(torch.int32) == 100 + i).all()
|
||||
# Layers within a group are disjoint.
|
||||
views["mla.0"][0].fill_(77)
|
||||
for i, name in enumerate(list(g1)[1:], start=1):
|
||||
assert (views[name][0].to(torch.int32) == i + 1).all()
|
||||
|
||||
def test_layer_compact_layout_rejected_for_overlaid_groups(self):
|
||||
# The layout has a single writer (backend-selection resolution); a layer-
|
||||
# compact layout reaching an overlay model's allocation is an error, not a
|
||||
# silent flip.
|
||||
groups, _, _ = _mixed_page_groups()
|
||||
with pytest.raises(
|
||||
ValueError, match="cannot express this model's mixed page sizes"
|
||||
):
|
||||
get_kv_cache_config_from_groups(_mock_vllm_config("LBNHC"), groups, MEMORY)
|
||||
|
||||
def test_unresolved_layout_rejected(self):
|
||||
groups, _, _ = _mixed_page_groups()
|
||||
with pytest.raises(ValueError, match="has not been resolved"):
|
||||
get_kv_cache_config_from_groups(_mock_vllm_config(None), groups, MEMORY)
|
||||
|
||||
def test_head_outer_layout_rejected_for_mixed_pages(self):
|
||||
groups, _, _ = _mixed_page_groups()
|
||||
with pytest.raises(
|
||||
ValueError, match="cannot express this model's mixed page sizes"
|
||||
):
|
||||
get_kv_cache_config_from_groups(_mock_vllm_config("LHBNC"), groups, MEMORY)
|
||||
|
||||
@pytest.mark.parametrize("layout", ["LBNHC", "BLHNC"])
|
||||
def test_bound_views_round_trip(self, layout):
|
||||
specs = {"mla.0": _mla(512), "mla.1": _mla(512), "idx.0": _mla(128)}
|
||||
groups = [_uniform_group(specs)]
|
||||
config = get_kv_cache_config_from_groups(
|
||||
_mock_vllm_config(), groups, available_memory=spec.page_size_bytes * 2 * 32
|
||||
_mock_vllm_config(layout), groups, MEMORY
|
||||
)
|
||||
|
||||
assert sum(t.size for t in config.kv_cache_tensors) == (
|
||||
spec.page_size_bytes * 2 * 32
|
||||
)
|
||||
assert [t.block_stride for t in config.kv_cache_tensors] == [0, 0]
|
||||
views = _bind(config, layout)
|
||||
for i, name in enumerate(specs):
|
||||
views[name].fill_(i + 1)
|
||||
for i, name in enumerate(specs):
|
||||
assert (views[name].to(torch.int32) == i + 1).all(), name
|
||||
assert views[name].shape[0] == config.num_blocks
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -124,7 +124,6 @@ def new_kv_cache_spec(
|
||||
page_size_padded=None,
|
||||
sliding_window=None,
|
||||
attention_chunk_size=None,
|
||||
indexes_kv_by_block_stride=False,
|
||||
kv_quant_mode=KVQuantMode.NONE,
|
||||
):
|
||||
return FullAttentionSpec(
|
||||
@@ -135,7 +134,6 @@ def new_kv_cache_spec(
|
||||
page_size_padded=page_size_padded,
|
||||
sliding_window=sliding_window,
|
||||
attention_chunk_size=attention_chunk_size,
|
||||
indexes_kv_by_block_stride=indexes_kv_by_block_stride,
|
||||
kv_quant_mode=kv_quant_mode,
|
||||
)
|
||||
|
||||
@@ -147,7 +145,6 @@ def new_sliding_window_spec(
|
||||
dtype=torch.float32,
|
||||
page_size_padded=None,
|
||||
sliding_window=1,
|
||||
indexes_kv_by_block_stride=False,
|
||||
):
|
||||
return SlidingWindowSpec(
|
||||
block_size=block_size,
|
||||
@@ -156,7 +153,6 @@ def new_sliding_window_spec(
|
||||
dtype=dtype,
|
||||
page_size_padded=page_size_padded,
|
||||
sliding_window=sliding_window,
|
||||
indexes_kv_by_block_stride=indexes_kv_by_block_stride,
|
||||
)
|
||||
|
||||
|
||||
@@ -875,6 +871,7 @@ def test_metrics_empty_stats():
|
||||
def test_get_kv_cache_configs_multiple_workers():
|
||||
model_config = ModelConfig(max_model_len=16)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
vllm_config.cache_config.kv_cache_layout = "LBNHC"
|
||||
vllm_config.cache_config.prefix_cache_retention_interval = None
|
||||
|
||||
ref_kv_cache_spec = new_kv_cache_spec()
|
||||
@@ -898,36 +895,21 @@ def test_get_kv_cache_configs_multiple_workers():
|
||||
ref_kv_cache_spec.page_size_bytes * 2 * 10,
|
||||
],
|
||||
)
|
||||
assert kv_cache_configs == [
|
||||
KVCacheConfig(
|
||||
num_blocks=10,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer1"]
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer2"]
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["layer1", "layer2"], ref_kv_cache_spec),
|
||||
],
|
||||
),
|
||||
KVCacheConfig(
|
||||
num_blocks=10,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer1"]
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer2"]
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["layer1", "layer2"], ref_kv_cache_spec),
|
||||
],
|
||||
),
|
||||
]
|
||||
expected = KVCacheConfig(
|
||||
num_blocks=10,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10 * 2,
|
||||
layers=["layer1", "layer2"],
|
||||
layer_stride=ref_kv_cache_spec.page_size_bytes * 10,
|
||||
block_stride=ref_kv_cache_spec.page_size_bytes,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["layer1", "layer2"], ref_kv_cache_spec),
|
||||
],
|
||||
)
|
||||
assert kv_cache_configs == [expected, expected]
|
||||
|
||||
# Different available memory. This is the case for TP.
|
||||
# Use the smallest memory available.
|
||||
@@ -939,36 +921,7 @@ def test_get_kv_cache_configs_multiple_workers():
|
||||
ref_kv_cache_spec.page_size_bytes * 2 * 20,
|
||||
],
|
||||
)
|
||||
assert kv_cache_configs == [
|
||||
KVCacheConfig(
|
||||
num_blocks=10,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer1"]
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer2"]
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["layer1", "layer2"], ref_kv_cache_spec),
|
||||
],
|
||||
),
|
||||
KVCacheConfig(
|
||||
num_blocks=10,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer1"]
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer2"]
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["layer1", "layer2"], ref_kv_cache_spec),
|
||||
],
|
||||
),
|
||||
]
|
||||
assert kv_cache_configs == [expected, expected]
|
||||
|
||||
# Different KV cache specs. This is the case for PP.
|
||||
different_layer_specs = [
|
||||
@@ -995,7 +948,10 @@ def test_get_kv_cache_configs_multiple_workers():
|
||||
num_blocks=10,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer1"]
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10,
|
||||
layers=["layer1"],
|
||||
layer_stride=ref_kv_cache_spec.page_size_bytes * 10,
|
||||
block_stride=ref_kv_cache_spec.page_size_bytes,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
@@ -1006,10 +962,10 @@ def test_get_kv_cache_configs_multiple_workers():
|
||||
num_blocks=10,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer2"]
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer3"]
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10 * 2,
|
||||
layers=["layer2", "layer3"],
|
||||
layer_stride=ref_kv_cache_spec.page_size_bytes * 10,
|
||||
block_stride=ref_kv_cache_spec.page_size_bytes,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
@@ -1046,57 +1002,39 @@ def test_get_kv_cache_configs_multiple_workers():
|
||||
ref_kv_cache_spec.page_size_bytes * 2 * 10,
|
||||
],
|
||||
)
|
||||
expected_12 = KVCacheConfig(
|
||||
num_blocks=10,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10 * 2,
|
||||
layers=["layer1", "layer2"],
|
||||
layer_stride=ref_kv_cache_spec.page_size_bytes * 10,
|
||||
block_stride=ref_kv_cache_spec.page_size_bytes,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["layer1", "layer2"], ref_kv_cache_spec),
|
||||
],
|
||||
)
|
||||
expected_3 = KVCacheConfig(
|
||||
num_blocks=10,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10,
|
||||
layers=["layer3"],
|
||||
layer_stride=ref_kv_cache_spec.page_size_bytes * 10,
|
||||
block_stride=ref_kv_cache_spec.page_size_bytes,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["layer3"], ref_kv_cache_spec),
|
||||
],
|
||||
)
|
||||
assert kv_cache_configs == [
|
||||
KVCacheConfig(
|
||||
num_blocks=10,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer1"]
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer2"]
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["layer1", "layer2"], ref_kv_cache_spec),
|
||||
],
|
||||
),
|
||||
KVCacheConfig(
|
||||
num_blocks=10,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer1"]
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer2"]
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["layer1", "layer2"], ref_kv_cache_spec),
|
||||
],
|
||||
),
|
||||
KVCacheConfig(
|
||||
num_blocks=10,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer3"]
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["layer3"], ref_kv_cache_spec),
|
||||
],
|
||||
),
|
||||
KVCacheConfig(
|
||||
num_blocks=10,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer3"]
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["layer3"], ref_kv_cache_spec),
|
||||
],
|
||||
),
|
||||
expected_12,
|
||||
expected_12,
|
||||
expected_3,
|
||||
expected_3,
|
||||
]
|
||||
|
||||
# Different workers have different types of layers. This is the case for
|
||||
@@ -1124,10 +1062,10 @@ def test_get_kv_cache_configs_multiple_workers():
|
||||
num_blocks=10,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer1"]
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer2"]
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10 * 2,
|
||||
layers=["layer1", "layer2"],
|
||||
layer_stride=ref_kv_cache_spec.page_size_bytes * 10,
|
||||
block_stride=ref_kv_cache_spec.page_size_bytes,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
@@ -1139,10 +1077,10 @@ def test_get_kv_cache_configs_multiple_workers():
|
||||
num_blocks=10,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer3"]
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer4"]
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10 * 2,
|
||||
layers=["layer3", "layer4"],
|
||||
layer_stride=ref_kv_cache_spec.page_size_bytes * 10,
|
||||
block_stride=ref_kv_cache_spec.page_size_bytes,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
@@ -1180,7 +1118,21 @@ def test_get_kv_cache_configs_multiple_workers():
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10,
|
||||
shared_by=["layer1", "layer2", "layer3"],
|
||||
layers=["layer1"],
|
||||
layer_stride=ref_kv_cache_spec.page_size_bytes * 10,
|
||||
block_stride=ref_kv_cache_spec.page_size_bytes,
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10,
|
||||
layers=["layer2"],
|
||||
layer_stride=ref_kv_cache_spec.page_size_bytes * 10,
|
||||
block_stride=ref_kv_cache_spec.page_size_bytes,
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10,
|
||||
layers=["layer3"],
|
||||
layer_stride=ref_kv_cache_spec.page_size_bytes * 10,
|
||||
block_stride=ref_kv_cache_spec.page_size_bytes,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
@@ -1194,7 +1146,21 @@ def test_get_kv_cache_configs_multiple_workers():
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10,
|
||||
shared_by=["layer4", "layer5", "layer6"],
|
||||
layers=["layer4"],
|
||||
layer_stride=ref_kv_cache_spec.page_size_bytes * 10,
|
||||
block_stride=ref_kv_cache_spec.page_size_bytes,
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10,
|
||||
layers=["layer5"],
|
||||
layer_stride=ref_kv_cache_spec.page_size_bytes * 10,
|
||||
block_stride=ref_kv_cache_spec.page_size_bytes,
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * 10,
|
||||
layers=["layer6"],
|
||||
layer_stride=ref_kv_cache_spec.page_size_bytes * 10,
|
||||
block_stride=ref_kv_cache_spec.page_size_bytes,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
@@ -1233,6 +1199,7 @@ def test_get_kv_cache_configs_multiple_workers():
|
||||
def test_get_kv_cache_configs_pp_sharding(asymmetric_memory):
|
||||
model_config = ModelConfig(max_model_len=512)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
vllm_config.cache_config.kv_cache_layout = "LBNHC"
|
||||
vllm_config.cache_config.prefix_cache_retention_interval = None
|
||||
|
||||
ref_kv_cache_spec = new_kv_cache_spec()
|
||||
@@ -1262,7 +1229,10 @@ def test_get_kv_cache_configs_pp_sharding(asymmetric_memory):
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * expected_num_blocks,
|
||||
shared_by=["layer1"],
|
||||
layers=["layer1"],
|
||||
layer_stride=ref_kv_cache_spec.page_size_bytes
|
||||
* expected_num_blocks,
|
||||
block_stride=ref_kv_cache_spec.page_size_bytes,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["layer1"], ref_kv_cache_spec)],
|
||||
@@ -1272,7 +1242,10 @@ def test_get_kv_cache_configs_pp_sharding(asymmetric_memory):
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * expected_num_blocks,
|
||||
shared_by=["layer2"],
|
||||
layers=["layer2"],
|
||||
layer_stride=ref_kv_cache_spec.page_size_bytes
|
||||
* expected_num_blocks,
|
||||
block_stride=ref_kv_cache_spec.page_size_bytes,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["layer2"], ref_kv_cache_spec)],
|
||||
@@ -1628,78 +1601,18 @@ def test_get_max_concurrency_for_kv_cache_config():
|
||||
)
|
||||
|
||||
|
||||
def test_get_max_concurrency_packed_kv_cache_config():
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
_get_kv_cache_config_packed,
|
||||
_use_packed_kv_cache_config,
|
||||
)
|
||||
|
||||
model_config = ModelConfig(
|
||||
"Qwen/Qwen1.5-7B",
|
||||
runner="generate",
|
||||
dtype="float16",
|
||||
max_model_len=16384,
|
||||
)
|
||||
scheduler_config = SchedulerConfig(
|
||||
max_num_batched_tokens=1024,
|
||||
enable_chunked_prefill=True,
|
||||
max_model_len=model_config.max_model_len,
|
||||
is_encoder_decoder=model_config.is_encoder_decoder,
|
||||
async_scheduling=False,
|
||||
)
|
||||
vllm_config = VllmConfig(
|
||||
model_config=model_config,
|
||||
scheduler_config=scheduler_config,
|
||||
)
|
||||
|
||||
# All-UniformTypeKVCacheSpecs groups select the packed layout.
|
||||
mla_specs = {f"layer_{i}": new_mla_spec() for i in range(4)}
|
||||
swa_specs = {
|
||||
f"layer_{i}": SlidingWindowMLASpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.float32,
|
||||
sliding_window=128,
|
||||
)
|
||||
for i in range(4, 6)
|
||||
}
|
||||
kv_cache_groups = [
|
||||
KVCacheGroupSpec(
|
||||
list(mla_specs),
|
||||
UniformTypeKVCacheSpecs(block_size=16, kv_cache_specs=mla_specs),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
list(swa_specs),
|
||||
UniformTypeKVCacheSpecs(block_size=16, kv_cache_specs=swa_specs),
|
||||
),
|
||||
]
|
||||
assert _use_packed_kv_cache_config(vllm_config, kv_cache_groups)
|
||||
num_blocks, kv_cache_tensors = _get_kv_cache_config_packed(
|
||||
vllm_config, kv_cache_groups, 2 * GiB_bytes
|
||||
)
|
||||
assert num_blocks > 0
|
||||
kv_cache_config_packed = KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=kv_cache_tensors,
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
)
|
||||
# Per-request blocks: the MLA group needs cdiv(16384, 16) = 1024 pages;
|
||||
# the SWA group cdiv(min(128 - 1 + 1024, 16384), 16) + 1 = 73. The
|
||||
# previous formula normalized by the first group's page size and gave
|
||||
# 1061 blocks per request instead of 1097.
|
||||
assert get_max_concurrency_for_kv_cache_config(
|
||||
vllm_config, kv_cache_config_packed
|
||||
) == num_blocks / (1024 + 73)
|
||||
|
||||
|
||||
def test_allocate_with_lookahead():
|
||||
"""Verify that lookahead tokens correctly affect block allocation"""
|
||||
block_size = 4
|
||||
config = KVCacheConfig(
|
||||
num_blocks=10,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(size=100, shared_by=["layer1"]),
|
||||
KVCacheTensor(
|
||||
size=100,
|
||||
layers=["layer1"],
|
||||
layer_stride=100,
|
||||
block_stride=10,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["layer1"], new_kv_cache_spec(block_size=block_size)),
|
||||
@@ -1763,6 +1676,7 @@ def test_get_kv_cache_config_one_worker():
|
||||
# pass max_model_len to pass check_enough_kv_cache_memory
|
||||
model_config = ModelConfig(max_model_len=16)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
vllm_config.cache_config.kv_cache_layout = "LBNHC"
|
||||
vllm_config.cache_config.prefix_cache_retention_interval = None
|
||||
|
||||
mem_per_block_per_layer = 16 * 2 * 64 * 4 * 2
|
||||
@@ -1778,8 +1692,12 @@ def test_get_kv_cache_config_one_worker():
|
||||
assert kv_cache_config_full == KVCacheConfig(
|
||||
num_blocks=32,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(size=mem_per_block_per_layer * 32, shared_by=["layer_1"]),
|
||||
KVCacheTensor(size=mem_per_block_per_layer * 32, shared_by=["layer_2"]),
|
||||
KVCacheTensor(
|
||||
size=mem_per_block_per_layer * 32 * 2,
|
||||
layers=["layer_1", "layer_2"],
|
||||
layer_stride=mem_per_block_per_layer * 32,
|
||||
block_stride=mem_per_block_per_layer,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["layer_1", "layer_2"], new_kv_cache_spec())],
|
||||
)
|
||||
@@ -1795,8 +1713,12 @@ def test_get_kv_cache_config_one_worker():
|
||||
assert kv_cache_config_sliding == KVCacheConfig(
|
||||
num_blocks=32,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(size=mem_per_block_per_layer * 32, shared_by=["layer_1"]),
|
||||
KVCacheTensor(size=mem_per_block_per_layer * 32, shared_by=["layer_2"]),
|
||||
KVCacheTensor(
|
||||
size=mem_per_block_per_layer * 32 * 2,
|
||||
layers=["layer_1", "layer_2"],
|
||||
layer_stride=mem_per_block_per_layer * 32,
|
||||
block_stride=mem_per_block_per_layer,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["layer_1", "layer_2"], new_sliding_window_spec())
|
||||
@@ -1815,8 +1737,12 @@ def test_get_kv_cache_config_one_worker():
|
||||
assert kv_cache_config_hybrid == KVCacheConfig(
|
||||
num_blocks=32,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(size=mem_per_block_per_layer * 32, shared_by=["layer_1"]),
|
||||
KVCacheTensor(size=mem_per_block_per_layer * 32, shared_by=["layer_2"]),
|
||||
KVCacheTensor(
|
||||
size=mem_per_block_per_layer * 32 * 2,
|
||||
layers=["layer_1", "layer_2"],
|
||||
layer_stride=mem_per_block_per_layer * 32,
|
||||
block_stride=mem_per_block_per_layer,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
@@ -1838,7 +1764,16 @@ def test_get_kv_cache_config_one_worker():
|
||||
num_blocks=64,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=mem_per_block_per_layer * 64, shared_by=["layer_1", "layer_2"]
|
||||
size=mem_per_block_per_layer * 64,
|
||||
layers=["layer_1"],
|
||||
layer_stride=mem_per_block_per_layer * 64,
|
||||
block_stride=mem_per_block_per_layer,
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=mem_per_block_per_layer * 64,
|
||||
layers=["layer_2"],
|
||||
layer_stride=mem_per_block_per_layer * 64,
|
||||
block_stride=mem_per_block_per_layer,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
@@ -1863,12 +1798,22 @@ def test_get_kv_cache_config_one_worker():
|
||||
num_blocks=32,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=mem_per_block_per_layer * 32,
|
||||
shared_by=["layer_1", "layer_3", "layer_4"],
|
||||
size=mem_per_block_per_layer * 32 * 2,
|
||||
layers=["layer_1", "layer_2"],
|
||||
layer_stride=mem_per_block_per_layer * 32,
|
||||
block_stride=mem_per_block_per_layer,
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=mem_per_block_per_layer * 32,
|
||||
shared_by=["layer_2", "layer_5", "layer_6"],
|
||||
size=mem_per_block_per_layer * 32 * 2,
|
||||
layers=["layer_3", "layer_5"],
|
||||
layer_stride=mem_per_block_per_layer * 32,
|
||||
block_stride=mem_per_block_per_layer,
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=mem_per_block_per_layer * 32 * 2,
|
||||
layers=["layer_4", "layer_6"],
|
||||
layer_stride=mem_per_block_per_layer * 32,
|
||||
block_stride=mem_per_block_per_layer,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
@@ -1898,15 +1843,28 @@ def test_get_kv_cache_config_one_worker():
|
||||
num_blocks=32,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=mem_per_block_per_layer * 32,
|
||||
shared_by=["layer_1", "layer_4", "layer_5", "layer_6"],
|
||||
size=mem_per_block_per_layer * 32 * 3,
|
||||
layers=["layer_1", "layer_2", "layer_3"],
|
||||
layer_stride=mem_per_block_per_layer * 32,
|
||||
block_stride=mem_per_block_per_layer,
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=mem_per_block_per_layer * 32,
|
||||
shared_by=["layer_2", "layer_7", "layer_8", "layer_9"],
|
||||
size=mem_per_block_per_layer * 32 * 3,
|
||||
layers=["layer_4", "layer_7", "layer_10"],
|
||||
layer_stride=mem_per_block_per_layer * 32,
|
||||
block_stride=mem_per_block_per_layer,
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=mem_per_block_per_layer * 32, shared_by=["layer_3", "layer_10"]
|
||||
size=mem_per_block_per_layer * 32 * 3,
|
||||
layers=["layer_5", "layer_8"],
|
||||
layer_stride=mem_per_block_per_layer * 32,
|
||||
block_stride=mem_per_block_per_layer,
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=mem_per_block_per_layer * 32 * 3,
|
||||
layers=["layer_6", "layer_9"],
|
||||
layer_stride=mem_per_block_per_layer * 32,
|
||||
block_stride=mem_per_block_per_layer,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
@@ -1943,28 +1901,23 @@ def test_get_kv_cache_config_one_worker():
|
||||
num_blocks=32,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=mem_per_block_per_layer * 32,
|
||||
shared_by=["layer_1", "layer_7"],
|
||||
size=mem_per_block_per_layer * 32 * 6,
|
||||
layers=[
|
||||
"layer_1",
|
||||
"layer_2",
|
||||
"layer_3",
|
||||
"layer_4",
|
||||
"layer_5",
|
||||
"layer_6",
|
||||
],
|
||||
layer_stride=mem_per_block_per_layer * 32,
|
||||
block_stride=mem_per_block_per_layer,
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=mem_per_block_per_layer * 32,
|
||||
shared_by=["layer_2", "layer_8"],
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=mem_per_block_per_layer * 32,
|
||||
shared_by=["layer_3", "layer_9"],
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=mem_per_block_per_layer * 32,
|
||||
shared_by=["layer_4", "layer_10"],
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=mem_per_block_per_layer * 32,
|
||||
shared_by=["layer_5", "layer_11"],
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=mem_per_block_per_layer * 32,
|
||||
shared_by=["layer_6"],
|
||||
size=mem_per_block_per_layer * 32 * 6,
|
||||
layers=["layer_7", "layer_8", "layer_9", "layer_10", "layer_11"],
|
||||
layer_stride=mem_per_block_per_layer * 32,
|
||||
block_stride=mem_per_block_per_layer,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
@@ -1987,11 +1940,24 @@ def test_get_kv_cache_config_one_worker():
|
||||
kv_cache_config_hybrid = get_kv_cache_configs(
|
||||
vllm_config, [kv_cache_specs_hybrid], [mem_per_block_per_layer * 3 * 32]
|
||||
)[0]
|
||||
# Layers of different page sizes pack densely into one allocation: the
|
||||
# 2x-larger layer_1 takes the first region, layer_2 the next.
|
||||
assert kv_cache_config_hybrid == KVCacheConfig(
|
||||
num_blocks=32,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(size=mem_per_block_per_layer * 32 * 2, shared_by=["layer_1"]),
|
||||
KVCacheTensor(size=mem_per_block_per_layer * 32, shared_by=["layer_2"]),
|
||||
KVCacheTensor(
|
||||
size=mem_per_block_per_layer * 32 * 3,
|
||||
layers=["layer_1"],
|
||||
layer_stride=mem_per_block_per_layer * 2 * 32,
|
||||
block_stride=mem_per_block_per_layer * 2,
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=mem_per_block_per_layer * 32 * 3,
|
||||
layers=["layer_2"],
|
||||
layer_stride=mem_per_block_per_layer * 32,
|
||||
block_stride=mem_per_block_per_layer,
|
||||
offset=mem_per_block_per_layer * 2 * 32,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
@@ -2015,7 +1981,16 @@ def test_get_kv_cache_config_one_worker():
|
||||
num_blocks=32,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=mem_per_block_per_layer * 32, shared_by=["layer_1", "layer_2"]
|
||||
size=mem_per_block_per_layer * 32,
|
||||
layers=["layer_1"],
|
||||
layer_stride=mem_per_block_per_layer * 32,
|
||||
block_stride=mem_per_block_per_layer,
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=mem_per_block_per_layer * 32,
|
||||
layers=["layer_2"],
|
||||
layer_stride=mem_per_block_per_layer * 32,
|
||||
block_stride=mem_per_block_per_layer,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
@@ -2028,9 +2003,9 @@ def test_get_kv_cache_config_one_worker():
|
||||
|
||||
# different hidden size that cannot be aligned by using different block size,
|
||||
# but can be aligned by padding the smaller physical page.
|
||||
swa_spec = new_sliding_window_spec(head_size=96, indexes_kv_by_block_stride=True)
|
||||
swa_spec = new_sliding_window_spec(head_size=96)
|
||||
kv_cache_specs_hybrid = {
|
||||
"layer_1": new_kv_cache_spec(head_size=64, indexes_kv_by_block_stride=True),
|
||||
"layer_1": new_kv_cache_spec(head_size=64),
|
||||
"layer_2": swa_spec,
|
||||
}
|
||||
|
||||
@@ -2041,7 +2016,18 @@ def test_get_kv_cache_config_one_worker():
|
||||
assert kv_cache_config_hybrid == KVCacheConfig(
|
||||
num_blocks=42,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(size=padded_page_size * 42, shared_by=["layer_1", "layer_2"]),
|
||||
KVCacheTensor(
|
||||
size=padded_page_size * 42,
|
||||
layers=["layer_1"],
|
||||
layer_stride=padded_page_size * 42,
|
||||
block_stride=padded_page_size,
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=padded_page_size * 42,
|
||||
layers=["layer_2"],
|
||||
layer_stride=padded_page_size * 42,
|
||||
block_stride=padded_page_size,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
@@ -2049,12 +2035,11 @@ def test_get_kv_cache_config_one_worker():
|
||||
new_kv_cache_spec(
|
||||
head_size=64,
|
||||
page_size_padded=padded_page_size,
|
||||
indexes_kv_by_block_stride=True,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["layer_2"],
|
||||
new_sliding_window_spec(head_size=96, indexes_kv_by_block_stride=True),
|
||||
new_sliding_window_spec(head_size=96),
|
||||
),
|
||||
],
|
||||
)
|
||||
@@ -2067,8 +2052,12 @@ def test_get_kv_cache_config_one_worker():
|
||||
assert kv_cache_config_override_blocks == KVCacheConfig(
|
||||
num_blocks=16,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(size=mem_per_block_per_layer * 16, shared_by=["layer_1"]),
|
||||
KVCacheTensor(size=mem_per_block_per_layer * 16, shared_by=["layer_2"]),
|
||||
KVCacheTensor(
|
||||
size=mem_per_block_per_layer * 16 * 2,
|
||||
layers=["layer_1", "layer_2"],
|
||||
layer_stride=mem_per_block_per_layer * 16,
|
||||
block_stride=mem_per_block_per_layer,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["layer_1", "layer_2"], new_kv_cache_spec())],
|
||||
)
|
||||
@@ -2319,6 +2308,45 @@ def test_hidden_state_group_preserves_hybrid_prefix_cache_granularity():
|
||||
) == (544, 136)
|
||||
|
||||
|
||||
def test_multi_run_layer_compact_strides_place_hoisted_heads():
|
||||
"""A layer-compact run region is its own dense allocation: under LHBNC the head
|
||||
groups sit between the layers and the blocks, so a run's block stride is one head
|
||||
group's slice, not the whole page (regression test for setStorage out-of-bounds
|
||||
on ROCm hybrid models)."""
|
||||
full = new_kv_cache_spec()
|
||||
swa = new_sliding_window_spec(sliding_window=full.block_size * 2)
|
||||
page = full.page_size_bytes
|
||||
assert swa.page_size_bytes == page
|
||||
|
||||
vllm_config = VllmConfig(model_config=ModelConfig(max_model_len=16))
|
||||
vllm_config.cache_config.kv_cache_layout = "LHBNC"
|
||||
config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
vllm_config,
|
||||
[
|
||||
KVCacheGroupSpec(["full.0", "full.1"], full),
|
||||
KVCacheGroupSpec(["swa.0", "swa.1"], swa),
|
||||
],
|
||||
available_memory=4 * page * 8,
|
||||
)
|
||||
|
||||
num_blocks = config.num_blocks
|
||||
head_group = full.block_size * full.state_content_size_bytes
|
||||
for tensor in config.kv_cache_tensors:
|
||||
# One head group per block, head groups between the layers and blocks.
|
||||
assert tensor.block_stride == head_group
|
||||
assert tensor.layer_stride == full.num_heads * num_blocks * head_group
|
||||
# The last block of the last layer's last head group must stay within the
|
||||
# allocation (the old page-based stride overran it).
|
||||
end = (
|
||||
tensor.offset
|
||||
+ tensor.layer_stride * (len(tensor.layers) - 1)
|
||||
+ (full.num_heads - 1) * num_blocks * head_group
|
||||
+ tensor.block_stride * (num_blocks - 1)
|
||||
+ head_group
|
||||
)
|
||||
assert end <= tensor.size
|
||||
|
||||
|
||||
def test_mla_draft_prefers_standard_layout_when_pages_can_be_unified():
|
||||
specs = {
|
||||
"target.0.attn": new_mla_spec(),
|
||||
@@ -2640,6 +2668,7 @@ def test_auto_fit_max_model_len():
|
||||
# Simulate the user passing -1 by setting original_max_model_len
|
||||
model_config.original_max_model_len = -1
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
vllm_config.cache_config.kv_cache_layout = "LBNHC"
|
||||
|
||||
mem_per_block_per_layer = 16 * 2 * 64 * 4 * 2 # 16KB per block per layer
|
||||
kv_cache_specs = {
|
||||
@@ -2658,6 +2687,7 @@ def test_auto_fit_max_model_len():
|
||||
model_config = ModelConfig(max_model_len=1024)
|
||||
model_config.original_max_model_len = -1
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
vllm_config.cache_config.kv_cache_layout = "LBNHC"
|
||||
|
||||
# With limited memory, max_model_len should be reduced
|
||||
# Need memory for at least max_model_len tokens
|
||||
@@ -2678,6 +2708,7 @@ def test_auto_fit_max_model_len_with_hybrid():
|
||||
# Simulate the user passing -1 by setting original_max_model_len
|
||||
model_config.original_max_model_len = -1
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
vllm_config.cache_config.kv_cache_layout = "LBNHC"
|
||||
|
||||
mem_per_block_per_layer = 16 * 2 * 64 * 4 * 2 # 16KB per block per layer
|
||||
gamma = 2
|
||||
@@ -2700,6 +2731,7 @@ def test_auto_fit_max_model_len_not_triggered():
|
||||
model_config = ModelConfig(max_model_len=16)
|
||||
# original_max_model_len should be None by default, not -1
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
vllm_config.cache_config.kv_cache_layout = "LBNHC"
|
||||
|
||||
mem_per_block_per_layer = 16 * 2 * 64 * 4 * 2
|
||||
kv_cache_specs = {
|
||||
@@ -2722,6 +2754,7 @@ def test_auto_fit_max_model_len_respects_num_gpu_blocks_override():
|
||||
model_config = ModelConfig(max_model_len=16384)
|
||||
model_config.original_max_model_len = -1 # request auto-fit
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
vllm_config.cache_config.kv_cache_layout = "LBNHC"
|
||||
# Cap the cache to 32 blocks regardless of available memory.
|
||||
vllm_config.cache_config.num_gpu_blocks_override = 32
|
||||
|
||||
@@ -2747,6 +2780,7 @@ def test_check_enough_kv_cache_memory_respects_num_gpu_blocks_override():
|
||||
"""
|
||||
model_config = ModelConfig(max_model_len=16384)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
vllm_config.cache_config.kv_cache_layout = "LBNHC"
|
||||
# 32 blocks is far too small for max_model_len=16384 (would need 1024).
|
||||
vllm_config.cache_config.num_gpu_blocks_override = 32
|
||||
|
||||
@@ -2769,14 +2803,13 @@ def test_unify_kv_cache_page_size_uses_padding_for_non_divisible_sizes():
|
||||
128-dim KV heads. The resulting page sizes are 3:2 rather than an integer
|
||||
block-size multiple, so the smaller page must be padded instead.
|
||||
"""
|
||||
# Both layers' backends opt into the padded-page strided view (e.g.
|
||||
# FlashAttention / its DiffKV subclass), so padding is allowed.
|
||||
# Attention layers read padded pages through a strided view, so padding is
|
||||
# allowed.
|
||||
target_spec = new_kv_cache_spec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=192,
|
||||
dtype=torch.bfloat16,
|
||||
indexes_kv_by_block_stride=True,
|
||||
)
|
||||
draft_spec = new_sliding_window_spec(
|
||||
block_size=16,
|
||||
@@ -2784,7 +2817,6 @@ def test_unify_kv_cache_page_size_uses_padding_for_non_divisible_sizes():
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=1024,
|
||||
indexes_kv_by_block_stride=True,
|
||||
)
|
||||
|
||||
unified_specs = kv_cache_utils.unify_kv_cache_spec_page_size(
|
||||
@@ -2802,35 +2834,6 @@ def test_unify_kv_cache_page_size_uses_padding_for_non_divisible_sizes():
|
||||
assert unified_draft_spec.page_size_bytes == target_spec.page_size_bytes
|
||||
|
||||
|
||||
def test_unify_kv_cache_page_size_padding_requires_backend_support():
|
||||
"""Padding is gated on the backend declaring ``indexes_kv_by_block_stride``.
|
||||
|
||||
A backend that does not support the strided padded-page view must raise
|
||||
rather than silently padding (and misreading KV at runtime).
|
||||
"""
|
||||
target_spec = new_kv_cache_spec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=192,
|
||||
dtype=torch.bfloat16,
|
||||
indexes_kv_by_block_stride=True,
|
||||
)
|
||||
# The non-divisible draft layer needs padding but its backend does not
|
||||
# support the strided padded-page view -> must raise, not silently pad.
|
||||
draft_spec = new_sliding_window_spec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=1024,
|
||||
indexes_kv_by_block_stride=False,
|
||||
)
|
||||
specs = {"target_attn": target_spec, "draft_attn": draft_spec}
|
||||
|
||||
with pytest.raises(NotImplementedError):
|
||||
kv_cache_utils.unify_kv_cache_spec_page_size(specs)
|
||||
|
||||
|
||||
def test_unpadded_page_size_includes_per_token_head_scales():
|
||||
# Per-token-head quant carries inline fp32 scales that are carved from the
|
||||
# raw KV allocation, so they must be budgeted into the offload width. The
|
||||
@@ -2850,10 +2853,14 @@ def test_unpadded_page_size_includes_per_token_head_scales():
|
||||
|
||||
def test_page_size_padded_wins():
|
||||
# An explicit padded page size takes precedence over the unpadded size.
|
||||
spec = new_kv_cache_spec(
|
||||
dtype=torch.uint8,
|
||||
kv_quant_mode=KVQuantMode.FP8_PER_TOKEN_HEAD,
|
||||
page_size_padded=65536,
|
||||
from vllm.v1.attention.backends.triton_attn import TritonAttentionBackend
|
||||
|
||||
spec = TritonAttentionBackend.customize_spec(
|
||||
new_kv_cache_spec(
|
||||
dtype=torch.uint8,
|
||||
kv_quant_mode=KVQuantMode.FP8_PER_TOKEN_HEAD,
|
||||
page_size_padded=65536,
|
||||
)
|
||||
)
|
||||
assert spec.page_size_bytes == 65536
|
||||
|
||||
@@ -2996,14 +3003,18 @@ def test_unify_kv_cache_spec_page_size_mamba():
|
||||
)
|
||||
assert unified["mamba_layer"].page_size_bytes == 32768
|
||||
|
||||
# 4. Attention layers with non-divisible page sizes still raise.
|
||||
with pytest.raises(NotImplementedError):
|
||||
kv_cache_utils.unify_kv_cache_spec_page_size(
|
||||
{
|
||||
"attn_layer": new_kv_cache_spec(block_size=24), # 24576
|
||||
"draft_attn_layer": draft_attn_spec, # 32768
|
||||
}
|
||||
)
|
||||
# 4. Attention layers with non-divisible page sizes are padded too: every
|
||||
# backend reads a padded page through the view's block stride, so there is
|
||||
# no longer a case that must raise.
|
||||
unified = kv_cache_utils.unify_kv_cache_spec_page_size(
|
||||
{
|
||||
"attn_layer": new_kv_cache_spec(block_size=24), # 24576
|
||||
"draft_attn_layer": draft_attn_spec, # 32768
|
||||
}
|
||||
)
|
||||
assert unified["attn_layer"].page_size_padded == 32768
|
||||
assert unified["attn_layer"].page_size_bytes == 32768
|
||||
assert unified["attn_layer"].block_size == 24
|
||||
|
||||
# 5. Uniform page sizes are returned unchanged.
|
||||
specs = {
|
||||
@@ -3135,6 +3146,7 @@ def test_auto_fit_max_model_len_reserves_null_block():
|
||||
model_config = ModelConfig(max_model_len=1024)
|
||||
model_config.original_max_model_len = -1
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
vllm_config.cache_config.kv_cache_layout = "LBNHC"
|
||||
spec = new_kv_cache_spec(block_size=block_size)
|
||||
|
||||
# Exactly the 1024 / 16 = 64 blocks a full-length request would need.
|
||||
|
||||
@@ -154,7 +154,7 @@ def make_kv_cache_config_hybrid_model(
|
||||
elif second_spec_type == "mamba":
|
||||
second_spec = MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=(1, 1),
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
)
|
||||
|
||||
@@ -189,7 +189,7 @@ def make_kv_cache_config_three_types(
|
||||
if third_spec_type == "mamba":
|
||||
third_spec = MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=(1, 1),
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
)
|
||||
elif third_spec_type == "sliding_window":
|
||||
@@ -768,12 +768,12 @@ def _make_hybrid_kv_cache_config(
|
||||
),
|
||||
"mamba": lambda: MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=(1, 1),
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
),
|
||||
"mamba_align": lambda: MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=(1, 1),
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
@@ -3273,7 +3273,7 @@ def test_hybrid_local_kv_retention_interval_survives_recycling():
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.uint8,
|
||||
compress_ratio=4,
|
||||
tokens_per_state=4,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
|
||||
Reference in New Issue
Block a user