[Bugfix][KV Offload] Register mixed page sizes in one cache group (#54756)

Signed-off-by: heliubj18 <chengfeng@ppio.com>
Co-authored-by: Kimi <noreply@moonshot.cn>
This commit is contained in:
He Liu
2026-09-07 20:50:15 -07:00
committed by GitHub
co-authored by Kimi
parent 9c297e3b8b
commit eb6b619ab2
3 changed files with 277 additions and 33 deletions
+21 -9
View File
@@ -426,6 +426,26 @@ class MockMambaBuilder(BaseMambaAttentionMetadataBuilder[BaseMambaAttentionMetad
return builder.build(0, common_metadata)
def dense_kv_cache_tensor(
raw: torch.Tensor,
spec: KVCacheSpec,
num_blocks: int,
num_layers: int,
layout: KVCacheLayout,
layer_names: list[str] | None = None,
) -> KVCacheTensor:
"""The ``KVCacheTensor`` for a dense allocation of ``num_layers`` layers."""
layer_stride, block_stride, _, _, _ = compute_layout_strides(
spec, num_blocks, num_layers, layout
)
return KVCacheTensor(
size=raw.numel(),
layers=layer_names or [str(i) for i in range(num_layers)],
layer_stride=layer_stride,
block_stride=block_stride,
)
def dense_kv_cache_views(
raw: torch.Tensor,
spec: KVCacheSpec,
@@ -435,15 +455,7 @@ def dense_kv_cache_views(
kernel_block_size: int | None = None,
) -> list[torch.Tensor]:
"""``create_kv_cache_views`` for a dense allocation of ``num_layers`` layers."""
layer_stride, block_stride, _, _, _ = compute_layout_strides(
spec, num_blocks, num_layers, layout
)
tensor = KVCacheTensor(
size=raw.numel(),
layers=[str(i) for i in range(num_layers)],
layer_stride=layer_stride,
block_stride=block_stride,
)
tensor = dense_kv_cache_tensor(raw, spec, num_blocks, num_layers, layout)
return create_kv_cache_views(
raw,
spec,
+215 -5
View File
@@ -20,8 +20,22 @@ from vllm.platforms import current_platform
if not current_platform.is_cuda_alike():
pytest.skip("Requires CUDA or ROCm", allow_module_level=True)
from tests.v1.attention.utils import dense_kv_cache_views
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheLayout
from tests.v1.attention.utils import dense_kv_cache_tensor, dense_kv_cache_views
from vllm.config import CacheConfig
from vllm.v1.core.kv_cache_utils import (
get_kv_cache_config_from_groups,
is_kv_cache_spec_uniform,
resolve_kv_cache_block_sizes,
)
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
KVCacheLayout,
KVCacheSpec,
KVCacheTensor,
MLAAttentionSpec,
UniformTypeKVCacheSpecs,
)
from vllm.v1.simple_kv_offload.copy_backend import DmaCopyBackend
from vllm.v1.simple_kv_offload.cuda_mem_ops import (
CU_MEMCPY_SRC_ACCESS_ORDER_ANY,
@@ -31,6 +45,7 @@ from vllm.v1.simple_kv_offload.cuda_mem_ops import (
)
from vllm.v1.simple_kv_offload.metadata import SimpleCPUOffloadMetadata
from vllm.v1.simple_kv_offload.worker import SimpleCPUOffloadWorker
from vllm.v1.worker.utils import allocate_kv_cache
NUM_BLOCKS = 64
BLOCK_BYTES = 4096
@@ -210,9 +225,14 @@ def test_register_shared_kv_cache_storage(monkeypatch, layout: KVCacheLayout):
device="cuda",
)
caches = dense_kv_cache_views(raw, spec, num_blocks, num_layers, layout)
layer_names = [f"layer.{i}" for i in range(num_layers)]
cache_config = MagicMock(
num_blocks=num_blocks,
kv_cache_tensors=[MagicMock(size=raw.nbytes)],
kv_cache_tensors=[
dense_kv_cache_tensor(
raw, spec, num_blocks, num_layers, layout, layer_names
)
],
)
worker = SimpleCPUOffloadWorker(
vllm_config=None,
@@ -252,7 +272,14 @@ def test_register_kv_cache_storage_with_trailing_padding(monkeypatch):
vllm_config=None,
kv_cache_config=MagicMock(
num_blocks=num_blocks,
kv_cache_tensors=[MagicMock(size=cache_bytes)],
kv_cache_tensors=[
KVCacheTensor(
size=cache_bytes,
layers=["layer.0"],
layer_stride=cache_bytes,
block_stride=block_bytes,
)
],
),
cpu_capacity_bytes=cache_bytes,
)
@@ -286,11 +313,16 @@ def test_register_separate_kv_head_groups(monkeypatch):
device="cuda",
)
caches = dense_kv_cache_views(raw, spec, num_blocks, num_layers, layout)
layer_names = [f"layer.{i}" for i in range(num_layers)]
worker = SimpleCPUOffloadWorker(
vllm_config=None,
kv_cache_config=MagicMock(
num_blocks=num_blocks,
kv_cache_tensors=[MagicMock(size=raw.nbytes)],
kv_cache_tensors=[
dense_kv_cache_tensor(
raw, spec, num_blocks, num_layers, layout, layer_names
)
],
),
cpu_capacity_bytes=raw.nbytes,
)
@@ -309,3 +341,181 @@ def test_register_separate_kv_head_groups(monkeypatch):
assert {cache.shape for cache in worker.gpu_kv_caches.values()} == {
(num_blocks, per_group_block_bytes)
}
def _dsa_specs(num_layers: int, block_size: int) -> dict[str, KVCacheSpec]:
"""A DSA model's per-layer specs (DeepSeek-V3.2, GLM-5.2).
Each decoder layer contributes an MLA latent cache and the indexer's key
cache, whose pages differ in size (``MLAAttentionSpec`` off ``head_size``:
512 + 64 for the latent, 128 fp8 keys + one fp32 scale per 128 elements for
the indexer). Both are ``MLAAttentionSpec`` with the same block size, so the
hybrid allocator puts them in one ``UniformTypeKVCacheSpecs`` cache group.
See ``DeepseekV32IndexerCache.get_kv_cache_spec`` and
``MLAAttention.get_kv_cache_spec``.
"""
specs: dict[str, KVCacheSpec] = {}
for i in range(num_layers):
specs[f"model.layers.{i}.self_attn.attn"] = MLAAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=512 + 64,
dtype=torch.uint8,
cache_dtype_str="fp8",
)
specs[f"model.layers.{i}.self_attn.indexer.k_cache"] = MLAAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=128 + 128 // 128 * 4,
dtype=torch.uint8,
)
return specs
def test_register_mixed_page_sizes_in_one_cache_group(monkeypatch):
"""Sparse-MLA models mix page sizes inside one cache group.
The MLA latent and indexer key caches of a layer have different page sizes,
so the allocation has no single per-layer block size to reinterpret it with:
``num_layers * (mla_page + indexer_page)`` is not a multiple of either page.
Registration must derive each region from the placement metadata instead.
"""
num_layers = 4
block_size = 64
specs = _dsa_specs(num_layers, block_size)
# Differing head sizes make the specs non-identical but same-type, which is
# what lands both caches of a layer in one group.
assert not is_kv_cache_spec_uniform(specs)
assert UniformTypeKVCacheSpecs.is_uniform_type(specs)
group = KVCacheGroupSpec(
list(specs),
UniformTypeKVCacheSpecs(block_size=block_size, kv_cache_specs=specs),
)
# DEEPSEEK_V32_INDEXER declares no supported layouts, so resolution lands on
# the default preference: layer-outermost, one region per layer.
layout = KVCacheLayout.LBNHC
vllm_config = MagicMock()
vllm_config.cache_config = CacheConfig()
vllm_config.cache_config.kv_cache_layout = layout.name
vllm_config.cache_config.num_gpu_blocks_override = None
pages = [spec.page_size_bytes for spec in specs.values()]
num_blocks = 4
kv_cache_config = get_kv_cache_config_from_groups(
vllm_config, [group], sum(pages) * num_blocks
)
assert kv_cache_config.num_blocks == num_blocks
assert len(set(pages)) > 1, "the mixed page sizes are what this test covers"
kv_caches = allocate_kv_cache(kv_cache_config, torch.device("cuda"), layout)
worker = SimpleCPUOffloadWorker(
vllm_config=None,
kv_cache_config=kv_cache_config,
cpu_capacity_bytes=sum(pages) * num_blocks,
)
worker._backend = MagicMock()
monkeypatch.setattr("vllm.v1.simple_kv_offload.worker.PIN_MEMORY", False)
worker.register_kv_caches(kv_caches)
assert worker.gpu_kv_caches is not None
# One region per layer, each with that layer's own page as its block stride.
assert sorted(worker.gpu_kv_caches) == sorted(specs)
assert {name: cache.shape[1] for name, cache in worker.gpu_kv_caches.items()} == {
name: spec.page_size_bytes for name, spec in specs.items()
}
assert all(cache.shape[0] == num_blocks for cache in worker.gpu_kv_caches.values())
# Every registered region must alias the bytes the model writes through, or
# offloaded blocks would be copied from the wrong place.
for name, cache in kv_caches.items():
region = worker.gpu_kv_caches[name]
assert region.data_ptr() == cache.data_ptr()
assert region.stride(0) == specs[name].page_size_bytes
@pytest.mark.parametrize("rank_blocks", [1, 5])
def test_register_mixed_page_sizes_odd_block_counts(monkeypatch, rank_blocks):
"""Registration holds at block counts that are not powers of two.
``num_blocks`` comes out of ``available_memory // bytes_per_block``, so
any integer is reachable. Registration derives regions from the config's
own block count and strides, and the CPU pool rounds down from the same
per-block byte total; none of it assumes anything about how the count
was reached.
"""
num_layers = 4
block_size = 64
specs = _dsa_specs(num_layers, block_size)
group = KVCacheGroupSpec(
list(specs),
UniformTypeKVCacheSpecs(block_size=block_size, kv_cache_specs=specs),
)
layout = KVCacheLayout.LBNHC
vllm_config = MagicMock()
vllm_config.cache_config = CacheConfig()
vllm_config.cache_config.kv_cache_layout = layout.name
vllm_config.cache_config.num_gpu_blocks_override = None
pages = [spec.page_size_bytes for spec in specs.values()]
kv_cache_config = get_kv_cache_config_from_groups(
vllm_config, [group], sum(pages) * rank_blocks
)
assert kv_cache_config.num_blocks == rank_blocks
kv_caches = allocate_kv_cache(kv_cache_config, torch.device("cuda"), layout)
worker = SimpleCPUOffloadWorker(
vllm_config=None,
kv_cache_config=kv_cache_config,
cpu_capacity_bytes=sum(pages) * rank_blocks,
)
worker._backend = MagicMock()
monkeypatch.setattr("vllm.v1.simple_kv_offload.worker.PIN_MEMORY", False)
worker.register_kv_caches(kv_caches)
assert worker.gpu_kv_caches is not None
assert {n: c.shape for n, c in worker.gpu_kv_caches.items()} == {
name: (rank_blocks, spec.page_size_bytes) for name, spec in specs.items()
}
assert worker.num_cpu_blocks == rank_blocks
def test_mixed_page_byte_placement_is_dcp_invariant():
"""DCP scales a block's token span, not its byte placement.
``get_kv_cache_config_from_groups`` never reads
``decode_context_parallel_size``, so every placement field registration
derives regions from is identical at dcp=1 and dcp=2. What DCP scales is
the block's token span, reported by ``resolve_kv_cache_block_sizes``;
checking that it doubles proves DCP was actually in effect, or the
identity above would be trivially true.
"""
specs = _dsa_specs(4, block_size=128)
placements = []
for dcp_size in (1, 2):
group = KVCacheGroupSpec(
list(specs),
UniformTypeKVCacheSpecs(block_size=128, kv_cache_specs=specs),
)
vllm_config = MagicMock()
vllm_config.cache_config = CacheConfig()
vllm_config.cache_config.block_size = 128
vllm_config.cache_config.kv_cache_layout = KVCacheLayout.LBNHC.name
vllm_config.cache_config.num_gpu_blocks_override = None
vllm_config.parallel_config.decode_context_parallel_size = dcp_size
page_bytes = sum(spec.page_size_bytes for spec in specs.values())
config = get_kv_cache_config_from_groups(vllm_config, [group], page_bytes * 4)
scheduler_block_size, _ = resolve_kv_cache_block_sizes(config, vllm_config)
assert scheduler_block_size == 128 * dcp_size
placements.append(
[
(t.size, tuple(t.layers), t.layer_stride, t.block_stride, t.offset)
for t in config.kv_cache_tensors
]
)
assert placements[0] == placements[1]
+41 -19
View File
@@ -100,33 +100,55 @@ class SimpleCPUOffloadWorker:
assert self.kv_cache_config is not None
num_blocks = self.kv_cache_config.num_blocks
assert self.kv_cache_config.kv_cache_tensors
logical_storage_bytes = self.kv_cache_config.kv_cache_tensors[0].size
# The DMA backend copies whole blocks as base + block_id * stride(0),
# so view each unique allocation as [num_blocks, block_bytes].
# The DMA backend copies blocks as base + block_id * stride(0), so every
# region is a [num_blocks, block_bytes] view. Block bytes come from each
# tensor's placement -- layers in one allocation can differ in page size.
unique_gpu_caches: dict[str, torch.Tensor] = {}
seen: set[tuple[torch.device, int]] = set()
for name, tensor in kv_caches.items():
storage = tensor.untyped_storage()
key = (tensor.device, storage.data_ptr())
if key in seen:
for kv_cache_tensor in self.kv_cache_config.kv_cache_tensors:
name = next((n for n in kv_cache_tensor.layers if n in kv_caches), None)
if name is None:
continue
seen.add(key)
num_layers = len(kv_cache_tensor.layers)
block_bytes = kv_cache_tensor.block_stride
blocks_span = num_blocks * block_bytes
layer_outermost = kv_cache_tensor.layer_stride >= blocks_span
if layer_outermost:
# One region per layer, or per KV head group under LHBNC.
start = kv_cache_tensor.offset
span = num_layers * kv_cache_tensor.layer_stride
layer_names = kv_cache_tensor.layers
else:
start = 0
span = blocks_span
layer_names = kv_cache_tensor.layers[:1]
physical_per_block, remainder = divmod(tensor.shape[0], num_blocks)
cache = kv_caches[name]
raw = torch.empty(0, dtype=torch.int8, device=cache.device).set_(
cache.untyped_storage()
)
assert raw.numel() >= start + span, (
f"KV cache {name!r} storage has {raw.numel()} bytes, smaller than "
f"the {span} bytes it places at offset {start}"
)
regions = raw[start : start + span].view(-1, num_blocks, block_bytes)
groups_per_layer, remainder = divmod(len(regions), len(layer_names))
assert remainder == 0, (
f"KV cache {name!r} has {tensor.shape[0]} physical blocks, which "
f"is not divisible by {num_blocks} scheduler blocks"
f"KV cache {name!r} places {len(layer_names)} layers in "
f"{len(regions)} block-strided regions"
)
block_bytes = tensor.stride(0) * tensor.element_size() * physical_per_block
raw = torch.empty(0, dtype=torch.int8, device=tensor.device).set_(storage)
assert raw.numel() >= logical_storage_bytes, (
f"KV cache {name!r} storage has {raw.numel()} bytes, smaller "
f"than the configured {logical_storage_bytes}-byte allocation"
)
regions = raw[:logical_storage_bytes].view(-1, num_blocks, block_bytes)
for idx, region in enumerate(regions):
key_name = name if len(regions) == 1 else f"{name}.{idx}"
key = (region.device, region.data_ptr())
if key in seen:
continue
seen.add(key)
layer_name = layer_names[idx // groups_per_layer]
key_name = (
layer_name
if groups_per_layer == 1
else f"{layer_name}.{idx % groups_per_layer}"
)
unique_gpu_caches[key_name] = region
# Compute per-tensor bytes_per_block. Tensors may have different