[6/N][KV-Cache Layout Refactor] Standardize KV cache layout (#51718)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
Lucas Wilkinson
2026-08-21 17:47:49 -07:00
committed by GitHub
parent 9ff7041b55
commit 8bdc70ec7b
150 changed files with 4006 additions and 5815 deletions
+175 -318
View File
@@ -1,380 +1,237 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for contiguous KV cache packing."""
"""Tests for contiguous KV cache packing.
Every cache group packs its layers densely into one block; groups overlay each other
(a block ID is owned by one group at a time), so the packed block stride is the
largest group's packing. The layout decides whether the layer dim sits outside the
block dim (a contiguous region per layer) or inside it (all layers' pages within each
block); the allocation is the same either way.
"""
from unittest.mock import MagicMock
import pytest
import torch
from vllm.config import CacheConfig
from vllm.v1.core.kv_cache_utils import (
_get_kv_cache_config_packed,
_get_kv_cache_groups_uniform_groups,
_get_kv_cache_bytes_per_block,
_pool_bytes_per_block,
get_kv_cache_config_from_groups,
)
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
KVCacheTensor,
KVCacheLayout,
MLAAttentionSpec,
SlidingWindowMLASpec,
SlidingWindowSpec,
UniformTypeKVCacheSpecs,
)
from vllm.v1.worker.utils import allocate_kv_cache
MEMORY = 8 * 1024 * 1024
def _make_mla_spec(page_size: int, block_size: int = 256) -> MLAAttentionSpec:
def _mla(head_size: int) -> MLAAttentionSpec:
return MLAAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=512,
dtype=torch.uint8,
page_size_padded=page_size,
cache_dtype_str="fp8_ds_mla",
model_version="deepseek_v4",
alignment=576,
block_size=64, num_kv_heads=1, head_size=head_size, dtype=torch.uint8
)
def _make_full_spec() -> FullAttentionSpec:
def _full() -> FullAttentionSpec:
return FullAttentionSpec(
block_size=16,
num_kv_heads=2,
head_size=64,
dtype=torch.float16,
block_size=16, num_kv_heads=2, head_size=64, dtype=torch.float16
)
def _make_sw_spec() -> SlidingWindowSpec:
return SlidingWindowSpec(
block_size=16,
num_kv_heads=2,
head_size=64,
dtype=torch.float16,
sliding_window=128,
def _uniform_group(specs: dict) -> KVCacheGroupSpec:
return KVCacheGroupSpec(
list(specs),
UniformTypeKVCacheSpecs(block_size=64, kv_cache_specs=specs),
)
def _make_groups(n_c4, n_c128, n_swa):
PS_C4_MLA = 37440
PS_C4_IDX = 8640
PS_C128 = 1728
PS_SWA = 37440
mla_specs = {}
for i in range(n_c4):
mla_specs[f"c4_mla.{i}"] = _make_mla_spec(PS_C4_MLA)
mla_specs[f"c4_idx.{i}"] = _make_mla_spec(PS_C4_IDX)
for i in range(n_c128):
mla_specs[f"c128_mla.{i}"] = _make_mla_spec(PS_C128)
mla_group = KVCacheGroupSpec(
layer_names=list(mla_specs.keys()),
kv_cache_spec=UniformTypeKVCacheSpecs(block_size=256, kv_cache_specs=mla_specs),
)
swa_specs = {}
for i in range(n_swa):
swa_specs[f"swa.{i}"] = _make_mla_spec(PS_SWA)
swa_group = KVCacheGroupSpec(
layer_names=list(swa_specs.keys()),
kv_cache_spec=UniformTypeKVCacheSpecs(block_size=256, kv_cache_specs=swa_specs),
)
return [mla_group, swa_group]
def _mixed_page_groups(n_mla=3, n_idx=3, n_swa=5):
"""A DeepSeek-V4-style hybrid: two groups with different page mixes."""
g1 = {f"mla.{i}": _mla(512) for i in range(n_mla)}
g1.update({f"idx.{i}": _mla(128) for i in range(n_idx)})
g2 = {f"swa.{i}": _mla(512) for i in range(n_swa)}
return [_uniform_group(g1), _uniform_group(g2)], g1, g2
def _mock_vllm_config(kv_connector_extra_config: dict[str, str] | None = None):
def _mock_vllm_config(layout: str | None):
config = MagicMock()
config.cache_config = CacheConfig()
config.cache_config.num_gpu_blocks_override = None
config.cache_config.prefix_cache_retention_interval = 0
config.kv_transfer_config = None
if kv_connector_extra_config is not None:
config.kv_transfer_config = MagicMock()
config.kv_transfer_config.kv_connector_extra_config = kv_connector_extra_config
config.cache_config.kv_cache_layout = layout
return config
def _run(n_c4=3, n_c128=2, n_swa=5, mem=100 * 1024 * 1024):
groups = _make_groups(n_c4, n_c128, n_swa)
return _get_kv_cache_config_packed(_mock_vllm_config(), groups, mem)
def _page_sizes_by_layer(
groups: list[KVCacheGroupSpec],
) -> dict[str, int]:
page_sizes = {}
for group in groups:
specs = group.kv_cache_spec.kv_cache_specs
for layer_name in group.layer_names:
page_sizes[layer_name] = specs[layer_name].page_size_bytes
return page_sizes
def _packing_by_layer(
tensors: list[KVCacheTensor],
) -> dict[str, tuple[int, int]]:
def _pages(groups) -> dict[str, int]:
return {
layer_name: (tensor.offset, tensor.block_stride)
for tensor in tensors
for layer_name in tensor.shared_by
name: group.kv_cache_spec.kv_cache_specs[name].page_size_bytes
if isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
else group.kv_cache_spec.page_size_bytes
for group in groups
for name in group.layer_names
}
def _make_views(
groups: list[KVCacheGroupSpec],
num_blocks: int,
tensors: list[KVCacheTensor],
) -> dict[str, torch.Tensor]:
page_sizes = _page_sizes_by_layer(groups)
packing = _packing_by_layer(tensors)
backing = torch.zeros(tensors[0].size, dtype=torch.uint8)
return {
layer_name: torch.as_strided(
backing,
size=(num_blocks, page_size),
stride=(packing[layer_name][1], 1),
storage_offset=packing[layer_name][0],
def _expected_bytes_per_block(groups) -> int:
pages = _pages(groups)
return max(sum(pages[n] for n in g.layer_names) for g in groups)
def _bind(config, layout: str):
return allocate_kv_cache(config, torch.device("cpu"), KVCacheLayout[layout], None)
class TestDensePacking:
def test_bytes_per_block_is_largest_group(self):
groups, g1, g2 = _mixed_page_groups()
assert _get_kv_cache_bytes_per_block(groups) == _expected_bytes_per_block(
groups
)
for layer_name, page_size in page_sizes.items()
}
def _make_page_group(prefix: str, page_sizes: list[int]) -> KVCacheGroupSpec:
specs = {
f"{prefix}.{i}": MagicMock(page_size_bytes=page_size)
for i, page_size in enumerate(page_sizes)
}
return KVCacheGroupSpec(
layer_names=list(specs),
kv_cache_spec=UniformTypeKVCacheSpecs(block_size=256, kv_cache_specs=specs),
)
class TestInterleavedPacking:
def test_compact_cache_overlays_fp32_state_group(self):
full_specs = {}
state_specs = {}
for i in range(2):
full_specs[f"mla.{i}"] = MLAAttentionSpec(
block_size=256,
num_kv_heads=1,
head_size=512,
dtype=torch.uint8,
page_size_padded=32768,
indexes_kv_by_block_stride=True,
compress_ratio=4,
)
full_specs[f"indexer.{i}"] = MLAAttentionSpec(
block_size=256,
num_kv_heads=1,
head_size=68,
dtype=torch.uint8,
page_size_padded=4608,
compress_ratio=4,
)
state_specs[f"mla_state.{i}"] = SlidingWindowMLASpec(
block_size=4,
num_kv_heads=1,
head_size=2048,
dtype=torch.float32,
sliding_window=8,
indexes_kv_by_block_stride=True,
)
state_specs[f"indexer_state.{i}"] = SlidingWindowMLASpec(
block_size=4,
num_kv_heads=1,
head_size=512,
dtype=torch.float32,
sliding_window=8,
indexes_kv_by_block_stride=True,
)
grouped_specs = [
UniformTypeKVCacheSpecs(block_size=256, kv_cache_specs=full_specs),
UniformTypeKVCacheSpecs(block_size=4, kv_cache_specs=state_specs),
config = get_kv_cache_config_from_groups(
_mock_vllm_config("BLHNC"), groups, MEMORY
)
# Groups overlay: both start at offset 0.
assert [tensor.offset for tensor in config.kv_cache_tensors].count(0) == 2
assert [tensor.layers for tensor in config.kv_cache_tensors] == [
list(g1)[:3],
list(g1)[3:],
list(g2),
]
groups = _get_kv_cache_groups_uniform_groups(grouped_specs)
assert len(groups) == 2
assert {full_specs[f"indexer.{i}"].page_size_bytes for i in range(2)} == {4608}
assert {full_specs[f"indexer.{i}"].real_page_size_bytes for i in range(2)} == {
4352
def test_layers_within_a_group_are_dense(self):
groups, _, _ = _mixed_page_groups()
pages = _pages(groups)
config = get_kv_cache_config_from_groups(
_mock_vllm_config("BLHNC"), groups, MEMORY
)
offsets = {
name: tensor.offset + i * tensor.layer_stride
for tensor in config.kv_cache_tensors
for i, name in enumerate(tensor.layers)
}
assert {
state_specs[f"indexer_state.{i}"].page_size_bytes for i in range(2)
} == {8192}
full_group_bytes = 2 * (32768 + 4608)
state_group_bytes = 2 * (32768 + 8192)
bytes_per_block = max(full_group_bytes, state_group_bytes)
num_blocks, tensors = _get_kv_cache_config_packed(
_mock_vllm_config(), groups, bytes_per_block * 32
)
assert num_blocks == 32
assert {tensor.block_stride for tensor in tensors} == {bytes_per_block}
packing = _packing_by_layer(tensors)
assert packing["mla.0"][0] == packing["mla_state.0"][0] == 0
assert packing["indexer.0"][0] == 32768
assert packing["indexer_state.0"][0] == 32768
def test_deepseek_v4_pro_stride(self):
groups = [
_make_page_group("full", [32768, 4608] * 30 + [1024] * 31),
_make_page_group("c4_state", [32768, 8192] * 30),
_make_page_group("c128_state", [32768] * 31),
_make_page_group("swa.0", [32768] * 31),
_make_page_group("swa.1", [32768] * 30),
]
expected_stride = 1_228_800
num_blocks, tensors = _get_kv_cache_config_packed(
_mock_vllm_config(), groups, expected_stride * 32
)
assert num_blocks == 32
assert {tensor.block_stride for tensor in tensors} == {expected_stride}
assert {tensor.size for tensor in tensors} == {expected_stride * 32}
def test_all_tensors_have_block_stride(self):
_, tensors = _run()
for t in tensors:
assert t.block_stride > 0
def test_all_tensors_share_same_size(self):
_, tensors = _run()
sizes = set(t.size for t in tensors)
assert len(sizes) == 1
assert sizes.pop() > 0
def test_offsets_within_one_block(self):
groups = _make_groups(n_c4=3, n_c128=2, n_swa=5)
_, tensors = _get_kv_cache_config_packed(
_mock_vllm_config(), groups, 100 * 1024 * 1024
)
page_sizes = _page_sizes_by_layer(groups)
packing = _packing_by_layer(tensors)
for layer_name, page_size in page_sizes.items():
offset, block_stride = packing[layer_name]
assert offset + page_size <= block_stride
def test_layouts_are_disjoint_within_each_group(self):
groups = _make_groups(n_c4=3, n_c128=2, n_swa=5)
_, tensors = _get_kv_cache_config_packed(
_mock_vllm_config(), groups, 100 * 1024 * 1024
)
page_sizes = _page_sizes_by_layer(groups)
packing = _packing_by_layer(tensors)
for group in groups:
ranges = sorted(
(packing[name][0], packing[name][0] + page_sizes[name])
for name in group.layer_names
expected = 0
for name in group.layer_names:
assert offsets[name] == expected
expected += pages[name]
@pytest.mark.parametrize("layout", ["LBNHC", "BLHNC"])
def test_allocation_is_layout_invariant(self, layout):
specs = {f"l.{i}": _full() for i in range(4)}
groups = [KVCacheGroupSpec(list(specs), _full())]
config = get_kv_cache_config_from_groups(
_mock_vllm_config(layout), groups, MEMORY
)
(tensor,) = config.kv_cache_tensors
page = _full().page_size_bytes
assert tensor.layers == list(specs)
assert config.num_blocks == MEMORY // (4 * page)
assert tensor.size == 4 * page * config.num_blocks
if layout == "LBNHC":
assert (tensor.layer_stride, tensor.block_stride) == (
page * config.num_blocks,
page,
)
assert all(left[1] <= right[0] for left, right in zip(ranges, ranges[1:]))
def test_all_layers_accounted_for(self):
n_c4, n_c128, n_swa = 5, 4, 7
_, tensors = _run(n_c4=n_c4, n_c128=n_c128, n_swa=n_swa)
all_names = set()
for t in tensors:
all_names.update(t.shared_by)
expected = n_c4 * 2 + n_c128 + n_swa
assert len(all_names) == expected
def test_group_owned_blocks_do_not_alias(self):
groups = _make_groups(n_c4=3, n_c128=2, n_swa=5)
num_blocks, tensors = _get_kv_cache_config_packed(
_mock_vllm_config(), groups, 100 * 1024 * 1024
)
views = _make_views(groups, num_blocks, tensors)
expected = {}
value = 1
for block_id, group in enumerate(groups):
for layer_name in group.layer_names:
views[layer_name][block_id].fill_(value)
expected[layer_name] = (block_id, value)
value += 1
for layer_name, (block_id, value) in expected.items():
assert (views[layer_name][block_id] == value).all()
# Once the first group releases its block, another group may reuse it.
for layer_name in groups[1].layer_names:
views[layer_name][0].fill_(255)
for layer_name in groups[1].layer_names:
assert (views[layer_name][0] == 255).all()
def test_hma_attention_groups_keep_default_backing(self):
full = _make_full_spec()
sw = _make_sw_spec()
page_size = full.page_size_bytes
groups = [
KVCacheGroupSpec(["full.0", "full.1"], full),
KVCacheGroupSpec(["sw.0", "sw.2"], sw),
KVCacheGroupSpec(["sw.1", "sw.3"], sw),
]
else:
assert (tensor.layer_stride, tensor.block_stride) == (page, 4 * page)
@pytest.mark.parametrize("layout", ["LBNHC", "BLHNC"])
def test_single_group_mixed_pages_follows_layout(self, layout):
specs = {"mla.0": _mla(512), "mla.1": _mla(512), "idx.0": _mla(128)}
groups = [_uniform_group(specs)]
config = get_kv_cache_config_from_groups(
_mock_vllm_config(), groups, available_memory=page_size * 2 * 32
_mock_vllm_config(layout), groups, MEMORY
)
block_stride = _expected_bytes_per_block(groups)
mla_tensor, idx_tensor = config.kv_cache_tensors
assert mla_tensor.layers == ["mla.0", "mla.1"]
assert idx_tensor.layers == ["idx.0"]
assert {t.size for t in config.kv_cache_tensors} == {
block_stride * config.num_blocks
}
if layout == "LBNHC":
assert (
idx_tensor.offset == 2 * _mla(512).page_size_bytes * config.num_blocks
)
assert mla_tensor.block_stride == _mla(512).page_size_bytes
else:
assert idx_tensor.offset == 2 * _mla(512).page_size_bytes
assert mla_tensor.block_stride == block_stride
assert config.num_blocks == 32
assert config.prefix_cache_retention_interval == 0
assert sum(t.size for t in config.kv_cache_tensors) == page_size * 2 * 32
assert config.kv_cache_tensors == [
KVCacheTensor(size=page_size * 32, shared_by=["full.0", "sw.0", "sw.1"]),
KVCacheTensor(size=page_size * 32, shared_by=["full.1", "sw.2", "sw.3"]),
]
def test_hma_attention_groups_use_packed_backing_with_enable_cross_layers(self):
full = _make_full_spec()
sw = _make_sw_spec()
page_size = full.page_size_bytes
groups = [
KVCacheGroupSpec(["full.0", "full.1"], full),
KVCacheGroupSpec(["sw.0", "sw.2"], sw),
KVCacheGroupSpec(["sw.1", "sw.3"], sw),
]
def test_overlaid_groups_alias_and_stay_isolated(self):
groups, g1, g2 = _mixed_page_groups()
# Overlay models resolve to a block-outer layout at backend selection (the
# model's backend declares it); mirror that here.
config = get_kv_cache_config_from_groups(
_mock_vllm_config({"enable_cross_layers_blocks": "True"}),
groups,
available_memory=page_size * 2 * 32,
_mock_vllm_config("BLNHC"), groups, MEMORY
)
assert config.num_blocks == MEMORY // _expected_bytes_per_block(groups)
assert _pool_bytes_per_block(groups) == _expected_bytes_per_block(groups)
assert config.num_blocks == 32
assert {t.size for t in config.kv_cache_tensors} == {page_size * 2 * 32}
assert config.kv_cache_tensors == [
KVCacheTensor(
size=page_size * 2 * 32,
shared_by=["full.0", "sw.0", "sw.1"],
offset=0,
block_stride=page_size * 2,
),
KVCacheTensor(
size=page_size * 2 * 32,
shared_by=["full.1", "sw.2", "sw.3"],
offset=page_size,
block_stride=page_size * 2,
),
]
views = _bind(config, "BLNHC")
assert set(views) == set(g1) | set(g2)
assert views["swa.0"].data_ptr() == views["mla.0"].data_ptr()
def test_single_group_attention_keeps_unpacked_layout(self):
spec = _make_full_spec()
groups = [KVCacheGroupSpec(["full.0", "full.1"], spec)]
# A block is owned by one group at a time: writes to group-owned
# blocks never disturb the other group's blocks.
for i, name in enumerate(g1):
views[name][0].fill_(i + 1)
views[name][2].fill_(i + 1)
for i, name in enumerate(g2):
views[name][1].fill_(100 + i)
views[name][3].fill_(100 + i)
for i, name in enumerate(g1):
assert (views[name][0].to(torch.int32) == i + 1).all()
assert (views[name][2].to(torch.int32) == i + 1).all()
for i, name in enumerate(g2):
assert (views[name][1].to(torch.int32) == 100 + i).all()
assert (views[name][3].to(torch.int32) == 100 + i).all()
# Layers within a group are disjoint.
views["mla.0"][0].fill_(77)
for i, name in enumerate(list(g1)[1:], start=1):
assert (views[name][0].to(torch.int32) == i + 1).all()
def test_layer_compact_layout_rejected_for_overlaid_groups(self):
# The layout has a single writer (backend-selection resolution); a layer-
# compact layout reaching an overlay model's allocation is an error, not a
# silent flip.
groups, _, _ = _mixed_page_groups()
with pytest.raises(
ValueError, match="cannot express this model's mixed page sizes"
):
get_kv_cache_config_from_groups(_mock_vllm_config("LBNHC"), groups, MEMORY)
def test_unresolved_layout_rejected(self):
groups, _, _ = _mixed_page_groups()
with pytest.raises(ValueError, match="has not been resolved"):
get_kv_cache_config_from_groups(_mock_vllm_config(None), groups, MEMORY)
def test_head_outer_layout_rejected_for_mixed_pages(self):
groups, _, _ = _mixed_page_groups()
with pytest.raises(
ValueError, match="cannot express this model's mixed page sizes"
):
get_kv_cache_config_from_groups(_mock_vllm_config("LHBNC"), groups, MEMORY)
@pytest.mark.parametrize("layout", ["LBNHC", "BLHNC"])
def test_bound_views_round_trip(self, layout):
specs = {"mla.0": _mla(512), "mla.1": _mla(512), "idx.0": _mla(128)}
groups = [_uniform_group(specs)]
config = get_kv_cache_config_from_groups(
_mock_vllm_config(), groups, available_memory=spec.page_size_bytes * 2 * 32
_mock_vllm_config(layout), groups, MEMORY
)
assert sum(t.size for t in config.kv_cache_tensors) == (
spec.page_size_bytes * 2 * 32
)
assert [t.block_stride for t in config.kv_cache_tensors] == [0, 0]
views = _bind(config, layout)
for i, name in enumerate(specs):
views[name].fill_(i + 1)
for i, name in enumerate(specs):
assert (views[name].to(torch.int32) == i + 1).all(), name
assert views[name].shape[0] == config.num_blocks
if __name__ == "__main__":
+300 -288
View File
@@ -124,7 +124,6 @@ def new_kv_cache_spec(
page_size_padded=None,
sliding_window=None,
attention_chunk_size=None,
indexes_kv_by_block_stride=False,
kv_quant_mode=KVQuantMode.NONE,
):
return FullAttentionSpec(
@@ -135,7 +134,6 @@ def new_kv_cache_spec(
page_size_padded=page_size_padded,
sliding_window=sliding_window,
attention_chunk_size=attention_chunk_size,
indexes_kv_by_block_stride=indexes_kv_by_block_stride,
kv_quant_mode=kv_quant_mode,
)
@@ -147,7 +145,6 @@ def new_sliding_window_spec(
dtype=torch.float32,
page_size_padded=None,
sliding_window=1,
indexes_kv_by_block_stride=False,
):
return SlidingWindowSpec(
block_size=block_size,
@@ -156,7 +153,6 @@ def new_sliding_window_spec(
dtype=dtype,
page_size_padded=page_size_padded,
sliding_window=sliding_window,
indexes_kv_by_block_stride=indexes_kv_by_block_stride,
)
@@ -875,6 +871,7 @@ def test_metrics_empty_stats():
def test_get_kv_cache_configs_multiple_workers():
model_config = ModelConfig(max_model_len=16)
vllm_config = VllmConfig(model_config=model_config)
vllm_config.cache_config.kv_cache_layout = "LBNHC"
vllm_config.cache_config.prefix_cache_retention_interval = None
ref_kv_cache_spec = new_kv_cache_spec()
@@ -898,36 +895,21 @@ def test_get_kv_cache_configs_multiple_workers():
ref_kv_cache_spec.page_size_bytes * 2 * 10,
],
)
assert kv_cache_configs == [
KVCacheConfig(
num_blocks=10,
kv_cache_tensors=[
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer1"]
),
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer2"]
),
],
kv_cache_groups=[
KVCacheGroupSpec(["layer1", "layer2"], ref_kv_cache_spec),
],
),
KVCacheConfig(
num_blocks=10,
kv_cache_tensors=[
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer1"]
),
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer2"]
),
],
kv_cache_groups=[
KVCacheGroupSpec(["layer1", "layer2"], ref_kv_cache_spec),
],
),
]
expected = KVCacheConfig(
num_blocks=10,
kv_cache_tensors=[
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * 10 * 2,
layers=["layer1", "layer2"],
layer_stride=ref_kv_cache_spec.page_size_bytes * 10,
block_stride=ref_kv_cache_spec.page_size_bytes,
),
],
kv_cache_groups=[
KVCacheGroupSpec(["layer1", "layer2"], ref_kv_cache_spec),
],
)
assert kv_cache_configs == [expected, expected]
# Different available memory. This is the case for TP.
# Use the smallest memory available.
@@ -939,36 +921,7 @@ def test_get_kv_cache_configs_multiple_workers():
ref_kv_cache_spec.page_size_bytes * 2 * 20,
],
)
assert kv_cache_configs == [
KVCacheConfig(
num_blocks=10,
kv_cache_tensors=[
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer1"]
),
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer2"]
),
],
kv_cache_groups=[
KVCacheGroupSpec(["layer1", "layer2"], ref_kv_cache_spec),
],
),
KVCacheConfig(
num_blocks=10,
kv_cache_tensors=[
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer1"]
),
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer2"]
),
],
kv_cache_groups=[
KVCacheGroupSpec(["layer1", "layer2"], ref_kv_cache_spec),
],
),
]
assert kv_cache_configs == [expected, expected]
# Different KV cache specs. This is the case for PP.
different_layer_specs = [
@@ -995,7 +948,10 @@ def test_get_kv_cache_configs_multiple_workers():
num_blocks=10,
kv_cache_tensors=[
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer1"]
size=ref_kv_cache_spec.page_size_bytes * 10,
layers=["layer1"],
layer_stride=ref_kv_cache_spec.page_size_bytes * 10,
block_stride=ref_kv_cache_spec.page_size_bytes,
),
],
kv_cache_groups=[
@@ -1006,10 +962,10 @@ def test_get_kv_cache_configs_multiple_workers():
num_blocks=10,
kv_cache_tensors=[
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer2"]
),
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer3"]
size=ref_kv_cache_spec.page_size_bytes * 10 * 2,
layers=["layer2", "layer3"],
layer_stride=ref_kv_cache_spec.page_size_bytes * 10,
block_stride=ref_kv_cache_spec.page_size_bytes,
),
],
kv_cache_groups=[
@@ -1046,57 +1002,39 @@ def test_get_kv_cache_configs_multiple_workers():
ref_kv_cache_spec.page_size_bytes * 2 * 10,
],
)
expected_12 = KVCacheConfig(
num_blocks=10,
kv_cache_tensors=[
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * 10 * 2,
layers=["layer1", "layer2"],
layer_stride=ref_kv_cache_spec.page_size_bytes * 10,
block_stride=ref_kv_cache_spec.page_size_bytes,
),
],
kv_cache_groups=[
KVCacheGroupSpec(["layer1", "layer2"], ref_kv_cache_spec),
],
)
expected_3 = KVCacheConfig(
num_blocks=10,
kv_cache_tensors=[
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * 10,
layers=["layer3"],
layer_stride=ref_kv_cache_spec.page_size_bytes * 10,
block_stride=ref_kv_cache_spec.page_size_bytes,
),
],
kv_cache_groups=[
KVCacheGroupSpec(["layer3"], ref_kv_cache_spec),
],
)
assert kv_cache_configs == [
KVCacheConfig(
num_blocks=10,
kv_cache_tensors=[
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer1"]
),
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer2"]
),
],
kv_cache_groups=[
KVCacheGroupSpec(["layer1", "layer2"], ref_kv_cache_spec),
],
),
KVCacheConfig(
num_blocks=10,
kv_cache_tensors=[
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer1"]
),
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer2"]
),
],
kv_cache_groups=[
KVCacheGroupSpec(["layer1", "layer2"], ref_kv_cache_spec),
],
),
KVCacheConfig(
num_blocks=10,
kv_cache_tensors=[
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer3"]
),
],
kv_cache_groups=[
KVCacheGroupSpec(["layer3"], ref_kv_cache_spec),
],
),
KVCacheConfig(
num_blocks=10,
kv_cache_tensors=[
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer3"]
),
],
kv_cache_groups=[
KVCacheGroupSpec(["layer3"], ref_kv_cache_spec),
],
),
expected_12,
expected_12,
expected_3,
expected_3,
]
# Different workers have different types of layers. This is the case for
@@ -1124,10 +1062,10 @@ def test_get_kv_cache_configs_multiple_workers():
num_blocks=10,
kv_cache_tensors=[
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer1"]
),
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer2"]
size=ref_kv_cache_spec.page_size_bytes * 10 * 2,
layers=["layer1", "layer2"],
layer_stride=ref_kv_cache_spec.page_size_bytes * 10,
block_stride=ref_kv_cache_spec.page_size_bytes,
),
],
kv_cache_groups=[
@@ -1139,10 +1077,10 @@ def test_get_kv_cache_configs_multiple_workers():
num_blocks=10,
kv_cache_tensors=[
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer3"]
),
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * 10, shared_by=["layer4"]
size=ref_kv_cache_spec.page_size_bytes * 10 * 2,
layers=["layer3", "layer4"],
layer_stride=ref_kv_cache_spec.page_size_bytes * 10,
block_stride=ref_kv_cache_spec.page_size_bytes,
),
],
kv_cache_groups=[
@@ -1180,7 +1118,21 @@ def test_get_kv_cache_configs_multiple_workers():
kv_cache_tensors=[
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * 10,
shared_by=["layer1", "layer2", "layer3"],
layers=["layer1"],
layer_stride=ref_kv_cache_spec.page_size_bytes * 10,
block_stride=ref_kv_cache_spec.page_size_bytes,
),
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * 10,
layers=["layer2"],
layer_stride=ref_kv_cache_spec.page_size_bytes * 10,
block_stride=ref_kv_cache_spec.page_size_bytes,
),
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * 10,
layers=["layer3"],
layer_stride=ref_kv_cache_spec.page_size_bytes * 10,
block_stride=ref_kv_cache_spec.page_size_bytes,
),
],
kv_cache_groups=[
@@ -1194,7 +1146,21 @@ def test_get_kv_cache_configs_multiple_workers():
kv_cache_tensors=[
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * 10,
shared_by=["layer4", "layer5", "layer6"],
layers=["layer4"],
layer_stride=ref_kv_cache_spec.page_size_bytes * 10,
block_stride=ref_kv_cache_spec.page_size_bytes,
),
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * 10,
layers=["layer5"],
layer_stride=ref_kv_cache_spec.page_size_bytes * 10,
block_stride=ref_kv_cache_spec.page_size_bytes,
),
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * 10,
layers=["layer6"],
layer_stride=ref_kv_cache_spec.page_size_bytes * 10,
block_stride=ref_kv_cache_spec.page_size_bytes,
),
],
kv_cache_groups=[
@@ -1233,6 +1199,7 @@ def test_get_kv_cache_configs_multiple_workers():
def test_get_kv_cache_configs_pp_sharding(asymmetric_memory):
model_config = ModelConfig(max_model_len=512)
vllm_config = VllmConfig(model_config=model_config)
vllm_config.cache_config.kv_cache_layout = "LBNHC"
vllm_config.cache_config.prefix_cache_retention_interval = None
ref_kv_cache_spec = new_kv_cache_spec()
@@ -1262,7 +1229,10 @@ def test_get_kv_cache_configs_pp_sharding(asymmetric_memory):
kv_cache_tensors=[
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * expected_num_blocks,
shared_by=["layer1"],
layers=["layer1"],
layer_stride=ref_kv_cache_spec.page_size_bytes
* expected_num_blocks,
block_stride=ref_kv_cache_spec.page_size_bytes,
),
],
kv_cache_groups=[KVCacheGroupSpec(["layer1"], ref_kv_cache_spec)],
@@ -1272,7 +1242,10 @@ def test_get_kv_cache_configs_pp_sharding(asymmetric_memory):
kv_cache_tensors=[
KVCacheTensor(
size=ref_kv_cache_spec.page_size_bytes * expected_num_blocks,
shared_by=["layer2"],
layers=["layer2"],
layer_stride=ref_kv_cache_spec.page_size_bytes
* expected_num_blocks,
block_stride=ref_kv_cache_spec.page_size_bytes,
),
],
kv_cache_groups=[KVCacheGroupSpec(["layer2"], ref_kv_cache_spec)],
@@ -1628,78 +1601,18 @@ def test_get_max_concurrency_for_kv_cache_config():
)
def test_get_max_concurrency_packed_kv_cache_config():
from vllm.v1.core.kv_cache_utils import (
_get_kv_cache_config_packed,
_use_packed_kv_cache_config,
)
model_config = ModelConfig(
"Qwen/Qwen1.5-7B",
runner="generate",
dtype="float16",
max_model_len=16384,
)
scheduler_config = SchedulerConfig(
max_num_batched_tokens=1024,
enable_chunked_prefill=True,
max_model_len=model_config.max_model_len,
is_encoder_decoder=model_config.is_encoder_decoder,
async_scheduling=False,
)
vllm_config = VllmConfig(
model_config=model_config,
scheduler_config=scheduler_config,
)
# All-UniformTypeKVCacheSpecs groups select the packed layout.
mla_specs = {f"layer_{i}": new_mla_spec() for i in range(4)}
swa_specs = {
f"layer_{i}": SlidingWindowMLASpec(
block_size=16,
num_kv_heads=1,
head_size=576,
dtype=torch.float32,
sliding_window=128,
)
for i in range(4, 6)
}
kv_cache_groups = [
KVCacheGroupSpec(
list(mla_specs),
UniformTypeKVCacheSpecs(block_size=16, kv_cache_specs=mla_specs),
),
KVCacheGroupSpec(
list(swa_specs),
UniformTypeKVCacheSpecs(block_size=16, kv_cache_specs=swa_specs),
),
]
assert _use_packed_kv_cache_config(vllm_config, kv_cache_groups)
num_blocks, kv_cache_tensors = _get_kv_cache_config_packed(
vllm_config, kv_cache_groups, 2 * GiB_bytes
)
assert num_blocks > 0
kv_cache_config_packed = KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=kv_cache_tensors,
kv_cache_groups=kv_cache_groups,
)
# Per-request blocks: the MLA group needs cdiv(16384, 16) = 1024 pages;
# the SWA group cdiv(min(128 - 1 + 1024, 16384), 16) + 1 = 73. The
# previous formula normalized by the first group's page size and gave
# 1061 blocks per request instead of 1097.
assert get_max_concurrency_for_kv_cache_config(
vllm_config, kv_cache_config_packed
) == num_blocks / (1024 + 73)
def test_allocate_with_lookahead():
"""Verify that lookahead tokens correctly affect block allocation"""
block_size = 4
config = KVCacheConfig(
num_blocks=10,
kv_cache_tensors=[
KVCacheTensor(size=100, shared_by=["layer1"]),
KVCacheTensor(
size=100,
layers=["layer1"],
layer_stride=100,
block_stride=10,
),
],
kv_cache_groups=[
KVCacheGroupSpec(["layer1"], new_kv_cache_spec(block_size=block_size)),
@@ -1763,6 +1676,7 @@ def test_get_kv_cache_config_one_worker():
# pass max_model_len to pass check_enough_kv_cache_memory
model_config = ModelConfig(max_model_len=16)
vllm_config = VllmConfig(model_config=model_config)
vllm_config.cache_config.kv_cache_layout = "LBNHC"
vllm_config.cache_config.prefix_cache_retention_interval = None
mem_per_block_per_layer = 16 * 2 * 64 * 4 * 2
@@ -1778,8 +1692,12 @@ def test_get_kv_cache_config_one_worker():
assert kv_cache_config_full == KVCacheConfig(
num_blocks=32,
kv_cache_tensors=[
KVCacheTensor(size=mem_per_block_per_layer * 32, shared_by=["layer_1"]),
KVCacheTensor(size=mem_per_block_per_layer * 32, shared_by=["layer_2"]),
KVCacheTensor(
size=mem_per_block_per_layer * 32 * 2,
layers=["layer_1", "layer_2"],
layer_stride=mem_per_block_per_layer * 32,
block_stride=mem_per_block_per_layer,
),
],
kv_cache_groups=[KVCacheGroupSpec(["layer_1", "layer_2"], new_kv_cache_spec())],
)
@@ -1795,8 +1713,12 @@ def test_get_kv_cache_config_one_worker():
assert kv_cache_config_sliding == KVCacheConfig(
num_blocks=32,
kv_cache_tensors=[
KVCacheTensor(size=mem_per_block_per_layer * 32, shared_by=["layer_1"]),
KVCacheTensor(size=mem_per_block_per_layer * 32, shared_by=["layer_2"]),
KVCacheTensor(
size=mem_per_block_per_layer * 32 * 2,
layers=["layer_1", "layer_2"],
layer_stride=mem_per_block_per_layer * 32,
block_stride=mem_per_block_per_layer,
),
],
kv_cache_groups=[
KVCacheGroupSpec(["layer_1", "layer_2"], new_sliding_window_spec())
@@ -1815,8 +1737,12 @@ def test_get_kv_cache_config_one_worker():
assert kv_cache_config_hybrid == KVCacheConfig(
num_blocks=32,
kv_cache_tensors=[
KVCacheTensor(size=mem_per_block_per_layer * 32, shared_by=["layer_1"]),
KVCacheTensor(size=mem_per_block_per_layer * 32, shared_by=["layer_2"]),
KVCacheTensor(
size=mem_per_block_per_layer * 32 * 2,
layers=["layer_1", "layer_2"],
layer_stride=mem_per_block_per_layer * 32,
block_stride=mem_per_block_per_layer,
),
],
kv_cache_groups=[
KVCacheGroupSpec(
@@ -1838,7 +1764,16 @@ def test_get_kv_cache_config_one_worker():
num_blocks=64,
kv_cache_tensors=[
KVCacheTensor(
size=mem_per_block_per_layer * 64, shared_by=["layer_1", "layer_2"]
size=mem_per_block_per_layer * 64,
layers=["layer_1"],
layer_stride=mem_per_block_per_layer * 64,
block_stride=mem_per_block_per_layer,
),
KVCacheTensor(
size=mem_per_block_per_layer * 64,
layers=["layer_2"],
layer_stride=mem_per_block_per_layer * 64,
block_stride=mem_per_block_per_layer,
),
],
kv_cache_groups=[
@@ -1863,12 +1798,22 @@ def test_get_kv_cache_config_one_worker():
num_blocks=32,
kv_cache_tensors=[
KVCacheTensor(
size=mem_per_block_per_layer * 32,
shared_by=["layer_1", "layer_3", "layer_4"],
size=mem_per_block_per_layer * 32 * 2,
layers=["layer_1", "layer_2"],
layer_stride=mem_per_block_per_layer * 32,
block_stride=mem_per_block_per_layer,
),
KVCacheTensor(
size=mem_per_block_per_layer * 32,
shared_by=["layer_2", "layer_5", "layer_6"],
size=mem_per_block_per_layer * 32 * 2,
layers=["layer_3", "layer_5"],
layer_stride=mem_per_block_per_layer * 32,
block_stride=mem_per_block_per_layer,
),
KVCacheTensor(
size=mem_per_block_per_layer * 32 * 2,
layers=["layer_4", "layer_6"],
layer_stride=mem_per_block_per_layer * 32,
block_stride=mem_per_block_per_layer,
),
],
kv_cache_groups=[
@@ -1898,15 +1843,28 @@ def test_get_kv_cache_config_one_worker():
num_blocks=32,
kv_cache_tensors=[
KVCacheTensor(
size=mem_per_block_per_layer * 32,
shared_by=["layer_1", "layer_4", "layer_5", "layer_6"],
size=mem_per_block_per_layer * 32 * 3,
layers=["layer_1", "layer_2", "layer_3"],
layer_stride=mem_per_block_per_layer * 32,
block_stride=mem_per_block_per_layer,
),
KVCacheTensor(
size=mem_per_block_per_layer * 32,
shared_by=["layer_2", "layer_7", "layer_8", "layer_9"],
size=mem_per_block_per_layer * 32 * 3,
layers=["layer_4", "layer_7", "layer_10"],
layer_stride=mem_per_block_per_layer * 32,
block_stride=mem_per_block_per_layer,
),
KVCacheTensor(
size=mem_per_block_per_layer * 32, shared_by=["layer_3", "layer_10"]
size=mem_per_block_per_layer * 32 * 3,
layers=["layer_5", "layer_8"],
layer_stride=mem_per_block_per_layer * 32,
block_stride=mem_per_block_per_layer,
),
KVCacheTensor(
size=mem_per_block_per_layer * 32 * 3,
layers=["layer_6", "layer_9"],
layer_stride=mem_per_block_per_layer * 32,
block_stride=mem_per_block_per_layer,
),
],
kv_cache_groups=[
@@ -1943,28 +1901,23 @@ def test_get_kv_cache_config_one_worker():
num_blocks=32,
kv_cache_tensors=[
KVCacheTensor(
size=mem_per_block_per_layer * 32,
shared_by=["layer_1", "layer_7"],
size=mem_per_block_per_layer * 32 * 6,
layers=[
"layer_1",
"layer_2",
"layer_3",
"layer_4",
"layer_5",
"layer_6",
],
layer_stride=mem_per_block_per_layer * 32,
block_stride=mem_per_block_per_layer,
),
KVCacheTensor(
size=mem_per_block_per_layer * 32,
shared_by=["layer_2", "layer_8"],
),
KVCacheTensor(
size=mem_per_block_per_layer * 32,
shared_by=["layer_3", "layer_9"],
),
KVCacheTensor(
size=mem_per_block_per_layer * 32,
shared_by=["layer_4", "layer_10"],
),
KVCacheTensor(
size=mem_per_block_per_layer * 32,
shared_by=["layer_5", "layer_11"],
),
KVCacheTensor(
size=mem_per_block_per_layer * 32,
shared_by=["layer_6"],
size=mem_per_block_per_layer * 32 * 6,
layers=["layer_7", "layer_8", "layer_9", "layer_10", "layer_11"],
layer_stride=mem_per_block_per_layer * 32,
block_stride=mem_per_block_per_layer,
),
],
kv_cache_groups=[
@@ -1987,11 +1940,24 @@ def test_get_kv_cache_config_one_worker():
kv_cache_config_hybrid = get_kv_cache_configs(
vllm_config, [kv_cache_specs_hybrid], [mem_per_block_per_layer * 3 * 32]
)[0]
# Layers of different page sizes pack densely into one allocation: the
# 2x-larger layer_1 takes the first region, layer_2 the next.
assert kv_cache_config_hybrid == KVCacheConfig(
num_blocks=32,
kv_cache_tensors=[
KVCacheTensor(size=mem_per_block_per_layer * 32 * 2, shared_by=["layer_1"]),
KVCacheTensor(size=mem_per_block_per_layer * 32, shared_by=["layer_2"]),
KVCacheTensor(
size=mem_per_block_per_layer * 32 * 3,
layers=["layer_1"],
layer_stride=mem_per_block_per_layer * 2 * 32,
block_stride=mem_per_block_per_layer * 2,
),
KVCacheTensor(
size=mem_per_block_per_layer * 32 * 3,
layers=["layer_2"],
layer_stride=mem_per_block_per_layer * 32,
block_stride=mem_per_block_per_layer,
offset=mem_per_block_per_layer * 2 * 32,
),
],
kv_cache_groups=[
KVCacheGroupSpec(
@@ -2015,7 +1981,16 @@ def test_get_kv_cache_config_one_worker():
num_blocks=32,
kv_cache_tensors=[
KVCacheTensor(
size=mem_per_block_per_layer * 32, shared_by=["layer_1", "layer_2"]
size=mem_per_block_per_layer * 32,
layers=["layer_1"],
layer_stride=mem_per_block_per_layer * 32,
block_stride=mem_per_block_per_layer,
),
KVCacheTensor(
size=mem_per_block_per_layer * 32,
layers=["layer_2"],
layer_stride=mem_per_block_per_layer * 32,
block_stride=mem_per_block_per_layer,
),
],
kv_cache_groups=[
@@ -2028,9 +2003,9 @@ def test_get_kv_cache_config_one_worker():
# different hidden size that cannot be aligned by using different block size,
# but can be aligned by padding the smaller physical page.
swa_spec = new_sliding_window_spec(head_size=96, indexes_kv_by_block_stride=True)
swa_spec = new_sliding_window_spec(head_size=96)
kv_cache_specs_hybrid = {
"layer_1": new_kv_cache_spec(head_size=64, indexes_kv_by_block_stride=True),
"layer_1": new_kv_cache_spec(head_size=64),
"layer_2": swa_spec,
}
@@ -2041,7 +2016,18 @@ def test_get_kv_cache_config_one_worker():
assert kv_cache_config_hybrid == KVCacheConfig(
num_blocks=42,
kv_cache_tensors=[
KVCacheTensor(size=padded_page_size * 42, shared_by=["layer_1", "layer_2"]),
KVCacheTensor(
size=padded_page_size * 42,
layers=["layer_1"],
layer_stride=padded_page_size * 42,
block_stride=padded_page_size,
),
KVCacheTensor(
size=padded_page_size * 42,
layers=["layer_2"],
layer_stride=padded_page_size * 42,
block_stride=padded_page_size,
),
],
kv_cache_groups=[
KVCacheGroupSpec(
@@ -2049,12 +2035,11 @@ def test_get_kv_cache_config_one_worker():
new_kv_cache_spec(
head_size=64,
page_size_padded=padded_page_size,
indexes_kv_by_block_stride=True,
),
),
KVCacheGroupSpec(
["layer_2"],
new_sliding_window_spec(head_size=96, indexes_kv_by_block_stride=True),
new_sliding_window_spec(head_size=96),
),
],
)
@@ -2067,8 +2052,12 @@ def test_get_kv_cache_config_one_worker():
assert kv_cache_config_override_blocks == KVCacheConfig(
num_blocks=16,
kv_cache_tensors=[
KVCacheTensor(size=mem_per_block_per_layer * 16, shared_by=["layer_1"]),
KVCacheTensor(size=mem_per_block_per_layer * 16, shared_by=["layer_2"]),
KVCacheTensor(
size=mem_per_block_per_layer * 16 * 2,
layers=["layer_1", "layer_2"],
layer_stride=mem_per_block_per_layer * 16,
block_stride=mem_per_block_per_layer,
),
],
kv_cache_groups=[KVCacheGroupSpec(["layer_1", "layer_2"], new_kv_cache_spec())],
)
@@ -2319,6 +2308,45 @@ def test_hidden_state_group_preserves_hybrid_prefix_cache_granularity():
) == (544, 136)
def test_multi_run_layer_compact_strides_place_hoisted_heads():
"""A layer-compact run region is its own dense allocation: under LHBNC the head
groups sit between the layers and the blocks, so a run's block stride is one head
group's slice, not the whole page (regression test for setStorage out-of-bounds
on ROCm hybrid models)."""
full = new_kv_cache_spec()
swa = new_sliding_window_spec(sliding_window=full.block_size * 2)
page = full.page_size_bytes
assert swa.page_size_bytes == page
vllm_config = VllmConfig(model_config=ModelConfig(max_model_len=16))
vllm_config.cache_config.kv_cache_layout = "LHBNC"
config = kv_cache_utils.get_kv_cache_config_from_groups(
vllm_config,
[
KVCacheGroupSpec(["full.0", "full.1"], full),
KVCacheGroupSpec(["swa.0", "swa.1"], swa),
],
available_memory=4 * page * 8,
)
num_blocks = config.num_blocks
head_group = full.block_size * full.state_content_size_bytes
for tensor in config.kv_cache_tensors:
# One head group per block, head groups between the layers and blocks.
assert tensor.block_stride == head_group
assert tensor.layer_stride == full.num_heads * num_blocks * head_group
# The last block of the last layer's last head group must stay within the
# allocation (the old page-based stride overran it).
end = (
tensor.offset
+ tensor.layer_stride * (len(tensor.layers) - 1)
+ (full.num_heads - 1) * num_blocks * head_group
+ tensor.block_stride * (num_blocks - 1)
+ head_group
)
assert end <= tensor.size
def test_mla_draft_prefers_standard_layout_when_pages_can_be_unified():
specs = {
"target.0.attn": new_mla_spec(),
@@ -2640,6 +2668,7 @@ def test_auto_fit_max_model_len():
# Simulate the user passing -1 by setting original_max_model_len
model_config.original_max_model_len = -1
vllm_config = VllmConfig(model_config=model_config)
vllm_config.cache_config.kv_cache_layout = "LBNHC"
mem_per_block_per_layer = 16 * 2 * 64 * 4 * 2 # 16KB per block per layer
kv_cache_specs = {
@@ -2658,6 +2687,7 @@ def test_auto_fit_max_model_len():
model_config = ModelConfig(max_model_len=1024)
model_config.original_max_model_len = -1
vllm_config = VllmConfig(model_config=model_config)
vllm_config.cache_config.kv_cache_layout = "LBNHC"
# With limited memory, max_model_len should be reduced
# Need memory for at least max_model_len tokens
@@ -2678,6 +2708,7 @@ def test_auto_fit_max_model_len_with_hybrid():
# Simulate the user passing -1 by setting original_max_model_len
model_config.original_max_model_len = -1
vllm_config = VllmConfig(model_config=model_config)
vllm_config.cache_config.kv_cache_layout = "LBNHC"
mem_per_block_per_layer = 16 * 2 * 64 * 4 * 2 # 16KB per block per layer
gamma = 2
@@ -2700,6 +2731,7 @@ def test_auto_fit_max_model_len_not_triggered():
model_config = ModelConfig(max_model_len=16)
# original_max_model_len should be None by default, not -1
vllm_config = VllmConfig(model_config=model_config)
vllm_config.cache_config.kv_cache_layout = "LBNHC"
mem_per_block_per_layer = 16 * 2 * 64 * 4 * 2
kv_cache_specs = {
@@ -2722,6 +2754,7 @@ def test_auto_fit_max_model_len_respects_num_gpu_blocks_override():
model_config = ModelConfig(max_model_len=16384)
model_config.original_max_model_len = -1 # request auto-fit
vllm_config = VllmConfig(model_config=model_config)
vllm_config.cache_config.kv_cache_layout = "LBNHC"
# Cap the cache to 32 blocks regardless of available memory.
vllm_config.cache_config.num_gpu_blocks_override = 32
@@ -2747,6 +2780,7 @@ def test_check_enough_kv_cache_memory_respects_num_gpu_blocks_override():
"""
model_config = ModelConfig(max_model_len=16384)
vllm_config = VllmConfig(model_config=model_config)
vllm_config.cache_config.kv_cache_layout = "LBNHC"
# 32 blocks is far too small for max_model_len=16384 (would need 1024).
vllm_config.cache_config.num_gpu_blocks_override = 32
@@ -2769,14 +2803,13 @@ def test_unify_kv_cache_page_size_uses_padding_for_non_divisible_sizes():
128-dim KV heads. The resulting page sizes are 3:2 rather than an integer
block-size multiple, so the smaller page must be padded instead.
"""
# Both layers' backends opt into the padded-page strided view (e.g.
# FlashAttention / its DiffKV subclass), so padding is allowed.
# Attention layers read padded pages through a strided view, so padding is
# allowed.
target_spec = new_kv_cache_spec(
block_size=16,
num_kv_heads=1,
head_size=192,
dtype=torch.bfloat16,
indexes_kv_by_block_stride=True,
)
draft_spec = new_sliding_window_spec(
block_size=16,
@@ -2784,7 +2817,6 @@ def test_unify_kv_cache_page_size_uses_padding_for_non_divisible_sizes():
head_size=128,
dtype=torch.bfloat16,
sliding_window=1024,
indexes_kv_by_block_stride=True,
)
unified_specs = kv_cache_utils.unify_kv_cache_spec_page_size(
@@ -2802,35 +2834,6 @@ def test_unify_kv_cache_page_size_uses_padding_for_non_divisible_sizes():
assert unified_draft_spec.page_size_bytes == target_spec.page_size_bytes
def test_unify_kv_cache_page_size_padding_requires_backend_support():
"""Padding is gated on the backend declaring ``indexes_kv_by_block_stride``.
A backend that does not support the strided padded-page view must raise
rather than silently padding (and misreading KV at runtime).
"""
target_spec = new_kv_cache_spec(
block_size=16,
num_kv_heads=1,
head_size=192,
dtype=torch.bfloat16,
indexes_kv_by_block_stride=True,
)
# The non-divisible draft layer needs padding but its backend does not
# support the strided padded-page view -> must raise, not silently pad.
draft_spec = new_sliding_window_spec(
block_size=16,
num_kv_heads=1,
head_size=128,
dtype=torch.bfloat16,
sliding_window=1024,
indexes_kv_by_block_stride=False,
)
specs = {"target_attn": target_spec, "draft_attn": draft_spec}
with pytest.raises(NotImplementedError):
kv_cache_utils.unify_kv_cache_spec_page_size(specs)
def test_unpadded_page_size_includes_per_token_head_scales():
# Per-token-head quant carries inline fp32 scales that are carved from the
# raw KV allocation, so they must be budgeted into the offload width. The
@@ -2850,10 +2853,14 @@ def test_unpadded_page_size_includes_per_token_head_scales():
def test_page_size_padded_wins():
# An explicit padded page size takes precedence over the unpadded size.
spec = new_kv_cache_spec(
dtype=torch.uint8,
kv_quant_mode=KVQuantMode.FP8_PER_TOKEN_HEAD,
page_size_padded=65536,
from vllm.v1.attention.backends.triton_attn import TritonAttentionBackend
spec = TritonAttentionBackend.customize_spec(
new_kv_cache_spec(
dtype=torch.uint8,
kv_quant_mode=KVQuantMode.FP8_PER_TOKEN_HEAD,
page_size_padded=65536,
)
)
assert spec.page_size_bytes == 65536
@@ -2996,14 +3003,18 @@ def test_unify_kv_cache_spec_page_size_mamba():
)
assert unified["mamba_layer"].page_size_bytes == 32768
# 4. Attention layers with non-divisible page sizes still raise.
with pytest.raises(NotImplementedError):
kv_cache_utils.unify_kv_cache_spec_page_size(
{
"attn_layer": new_kv_cache_spec(block_size=24), # 24576
"draft_attn_layer": draft_attn_spec, # 32768
}
)
# 4. Attention layers with non-divisible page sizes are padded too: every
# backend reads a padded page through the view's block stride, so there is
# no longer a case that must raise.
unified = kv_cache_utils.unify_kv_cache_spec_page_size(
{
"attn_layer": new_kv_cache_spec(block_size=24), # 24576
"draft_attn_layer": draft_attn_spec, # 32768
}
)
assert unified["attn_layer"].page_size_padded == 32768
assert unified["attn_layer"].page_size_bytes == 32768
assert unified["attn_layer"].block_size == 24
# 5. Uniform page sizes are returned unchanged.
specs = {
@@ -3135,6 +3146,7 @@ def test_auto_fit_max_model_len_reserves_null_block():
model_config = ModelConfig(max_model_len=1024)
model_config.original_max_model_len = -1
vllm_config = VllmConfig(model_config=model_config)
vllm_config.cache_config.kv_cache_layout = "LBNHC"
spec = new_kv_cache_spec(block_size=block_size)
# Exactly the 1024 / 16 = 64 blocks a full-length request would need.
+5 -5
View File
@@ -154,7 +154,7 @@ def make_kv_cache_config_hybrid_model(
elif second_spec_type == "mamba":
second_spec = MambaSpec(
block_size=block_size,
shapes=(1, 1),
shapes=((1, 1),),
dtypes=(torch.float32,),
)
@@ -189,7 +189,7 @@ def make_kv_cache_config_three_types(
if third_spec_type == "mamba":
third_spec = MambaSpec(
block_size=block_size,
shapes=(1, 1),
shapes=((1, 1),),
dtypes=(torch.float32,),
)
elif third_spec_type == "sliding_window":
@@ -768,12 +768,12 @@ def _make_hybrid_kv_cache_config(
),
"mamba": lambda: MambaSpec(
block_size=block_size,
shapes=(1, 1),
shapes=((1, 1),),
dtypes=(torch.float32,),
),
"mamba_align": lambda: MambaSpec(
block_size=block_size,
shapes=(1, 1),
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
@@ -3273,7 +3273,7 @@ def test_hybrid_local_kv_retention_interval_survives_recycling():
num_kv_heads=1,
head_size=1,
dtype=torch.uint8,
compress_ratio=4,
tokens_per_state=4,
),
),
KVCacheGroupSpec(