[Bugfix][KV Offload] Register mixed page sizes in one cache group (#54756)
Signed-off-by: heliubj18 <chengfeng@ppio.com> Co-authored-by: Kimi <noreply@moonshot.cn>
This commit is contained in:
@@ -426,6 +426,26 @@ class MockMambaBuilder(BaseMambaAttentionMetadataBuilder[BaseMambaAttentionMetad
|
||||
return builder.build(0, common_metadata)
|
||||
|
||||
|
||||
def dense_kv_cache_tensor(
|
||||
raw: torch.Tensor,
|
||||
spec: KVCacheSpec,
|
||||
num_blocks: int,
|
||||
num_layers: int,
|
||||
layout: KVCacheLayout,
|
||||
layer_names: list[str] | None = None,
|
||||
) -> KVCacheTensor:
|
||||
"""The ``KVCacheTensor`` for a dense allocation of ``num_layers`` layers."""
|
||||
layer_stride, block_stride, _, _, _ = compute_layout_strides(
|
||||
spec, num_blocks, num_layers, layout
|
||||
)
|
||||
return KVCacheTensor(
|
||||
size=raw.numel(),
|
||||
layers=layer_names or [str(i) for i in range(num_layers)],
|
||||
layer_stride=layer_stride,
|
||||
block_stride=block_stride,
|
||||
)
|
||||
|
||||
|
||||
def dense_kv_cache_views(
|
||||
raw: torch.Tensor,
|
||||
spec: KVCacheSpec,
|
||||
@@ -435,15 +455,7 @@ def dense_kv_cache_views(
|
||||
kernel_block_size: int | None = None,
|
||||
) -> list[torch.Tensor]:
|
||||
"""``create_kv_cache_views`` for a dense allocation of ``num_layers`` layers."""
|
||||
layer_stride, block_stride, _, _, _ = compute_layout_strides(
|
||||
spec, num_blocks, num_layers, layout
|
||||
)
|
||||
tensor = KVCacheTensor(
|
||||
size=raw.numel(),
|
||||
layers=[str(i) for i in range(num_layers)],
|
||||
layer_stride=layer_stride,
|
||||
block_stride=block_stride,
|
||||
)
|
||||
tensor = dense_kv_cache_tensor(raw, spec, num_blocks, num_layers, layout)
|
||||
return create_kv_cache_views(
|
||||
raw,
|
||||
spec,
|
||||
|
||||
@@ -20,8 +20,22 @@ from vllm.platforms import current_platform
|
||||
if not current_platform.is_cuda_alike():
|
||||
pytest.skip("Requires CUDA or ROCm", allow_module_level=True)
|
||||
|
||||
from tests.v1.attention.utils import dense_kv_cache_views
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheLayout
|
||||
from tests.v1.attention.utils import dense_kv_cache_tensor, dense_kv_cache_views
|
||||
from vllm.config import CacheConfig
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
get_kv_cache_config_from_groups,
|
||||
is_kv_cache_spec_uniform,
|
||||
resolve_kv_cache_block_sizes,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheLayout,
|
||||
KVCacheSpec,
|
||||
KVCacheTensor,
|
||||
MLAAttentionSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
from vllm.v1.simple_kv_offload.copy_backend import DmaCopyBackend
|
||||
from vllm.v1.simple_kv_offload.cuda_mem_ops import (
|
||||
CU_MEMCPY_SRC_ACCESS_ORDER_ANY,
|
||||
@@ -31,6 +45,7 @@ from vllm.v1.simple_kv_offload.cuda_mem_ops import (
|
||||
)
|
||||
from vllm.v1.simple_kv_offload.metadata import SimpleCPUOffloadMetadata
|
||||
from vllm.v1.simple_kv_offload.worker import SimpleCPUOffloadWorker
|
||||
from vllm.v1.worker.utils import allocate_kv_cache
|
||||
|
||||
NUM_BLOCKS = 64
|
||||
BLOCK_BYTES = 4096
|
||||
@@ -210,9 +225,14 @@ def test_register_shared_kv_cache_storage(monkeypatch, layout: KVCacheLayout):
|
||||
device="cuda",
|
||||
)
|
||||
caches = dense_kv_cache_views(raw, spec, num_blocks, num_layers, layout)
|
||||
layer_names = [f"layer.{i}" for i in range(num_layers)]
|
||||
cache_config = MagicMock(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[MagicMock(size=raw.nbytes)],
|
||||
kv_cache_tensors=[
|
||||
dense_kv_cache_tensor(
|
||||
raw, spec, num_blocks, num_layers, layout, layer_names
|
||||
)
|
||||
],
|
||||
)
|
||||
worker = SimpleCPUOffloadWorker(
|
||||
vllm_config=None,
|
||||
@@ -252,7 +272,14 @@ def test_register_kv_cache_storage_with_trailing_padding(monkeypatch):
|
||||
vllm_config=None,
|
||||
kv_cache_config=MagicMock(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[MagicMock(size=cache_bytes)],
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=cache_bytes,
|
||||
layers=["layer.0"],
|
||||
layer_stride=cache_bytes,
|
||||
block_stride=block_bytes,
|
||||
)
|
||||
],
|
||||
),
|
||||
cpu_capacity_bytes=cache_bytes,
|
||||
)
|
||||
@@ -286,11 +313,16 @@ def test_register_separate_kv_head_groups(monkeypatch):
|
||||
device="cuda",
|
||||
)
|
||||
caches = dense_kv_cache_views(raw, spec, num_blocks, num_layers, layout)
|
||||
layer_names = [f"layer.{i}" for i in range(num_layers)]
|
||||
worker = SimpleCPUOffloadWorker(
|
||||
vllm_config=None,
|
||||
kv_cache_config=MagicMock(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[MagicMock(size=raw.nbytes)],
|
||||
kv_cache_tensors=[
|
||||
dense_kv_cache_tensor(
|
||||
raw, spec, num_blocks, num_layers, layout, layer_names
|
||||
)
|
||||
],
|
||||
),
|
||||
cpu_capacity_bytes=raw.nbytes,
|
||||
)
|
||||
@@ -309,3 +341,181 @@ def test_register_separate_kv_head_groups(monkeypatch):
|
||||
assert {cache.shape for cache in worker.gpu_kv_caches.values()} == {
|
||||
(num_blocks, per_group_block_bytes)
|
||||
}
|
||||
|
||||
|
||||
def _dsa_specs(num_layers: int, block_size: int) -> dict[str, KVCacheSpec]:
|
||||
"""A DSA model's per-layer specs (DeepSeek-V3.2, GLM-5.2).
|
||||
|
||||
Each decoder layer contributes an MLA latent cache and the indexer's key
|
||||
cache, whose pages differ in size (``MLAAttentionSpec`` off ``head_size``:
|
||||
512 + 64 for the latent, 128 fp8 keys + one fp32 scale per 128 elements for
|
||||
the indexer). Both are ``MLAAttentionSpec`` with the same block size, so the
|
||||
hybrid allocator puts them in one ``UniformTypeKVCacheSpecs`` cache group.
|
||||
See ``DeepseekV32IndexerCache.get_kv_cache_spec`` and
|
||||
``MLAAttention.get_kv_cache_spec``.
|
||||
"""
|
||||
specs: dict[str, KVCacheSpec] = {}
|
||||
for i in range(num_layers):
|
||||
specs[f"model.layers.{i}.self_attn.attn"] = MLAAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=512 + 64,
|
||||
dtype=torch.uint8,
|
||||
cache_dtype_str="fp8",
|
||||
)
|
||||
specs[f"model.layers.{i}.self_attn.indexer.k_cache"] = MLAAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=128 + 128 // 128 * 4,
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
return specs
|
||||
|
||||
|
||||
def test_register_mixed_page_sizes_in_one_cache_group(monkeypatch):
|
||||
"""Sparse-MLA models mix page sizes inside one cache group.
|
||||
|
||||
The MLA latent and indexer key caches of a layer have different page sizes,
|
||||
so the allocation has no single per-layer block size to reinterpret it with:
|
||||
``num_layers * (mla_page + indexer_page)`` is not a multiple of either page.
|
||||
Registration must derive each region from the placement metadata instead.
|
||||
"""
|
||||
num_layers = 4
|
||||
block_size = 64
|
||||
specs = _dsa_specs(num_layers, block_size)
|
||||
# Differing head sizes make the specs non-identical but same-type, which is
|
||||
# what lands both caches of a layer in one group.
|
||||
assert not is_kv_cache_spec_uniform(specs)
|
||||
assert UniformTypeKVCacheSpecs.is_uniform_type(specs)
|
||||
group = KVCacheGroupSpec(
|
||||
list(specs),
|
||||
UniformTypeKVCacheSpecs(block_size=block_size, kv_cache_specs=specs),
|
||||
)
|
||||
|
||||
# DEEPSEEK_V32_INDEXER declares no supported layouts, so resolution lands on
|
||||
# the default preference: layer-outermost, one region per layer.
|
||||
layout = KVCacheLayout.LBNHC
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.cache_config = CacheConfig()
|
||||
vllm_config.cache_config.kv_cache_layout = layout.name
|
||||
vllm_config.cache_config.num_gpu_blocks_override = None
|
||||
|
||||
pages = [spec.page_size_bytes for spec in specs.values()]
|
||||
num_blocks = 4
|
||||
kv_cache_config = get_kv_cache_config_from_groups(
|
||||
vllm_config, [group], sum(pages) * num_blocks
|
||||
)
|
||||
assert kv_cache_config.num_blocks == num_blocks
|
||||
assert len(set(pages)) > 1, "the mixed page sizes are what this test covers"
|
||||
|
||||
kv_caches = allocate_kv_cache(kv_cache_config, torch.device("cuda"), layout)
|
||||
worker = SimpleCPUOffloadWorker(
|
||||
vllm_config=None,
|
||||
kv_cache_config=kv_cache_config,
|
||||
cpu_capacity_bytes=sum(pages) * num_blocks,
|
||||
)
|
||||
worker._backend = MagicMock()
|
||||
monkeypatch.setattr("vllm.v1.simple_kv_offload.worker.PIN_MEMORY", False)
|
||||
|
||||
worker.register_kv_caches(kv_caches)
|
||||
|
||||
assert worker.gpu_kv_caches is not None
|
||||
# One region per layer, each with that layer's own page as its block stride.
|
||||
assert sorted(worker.gpu_kv_caches) == sorted(specs)
|
||||
assert {name: cache.shape[1] for name, cache in worker.gpu_kv_caches.items()} == {
|
||||
name: spec.page_size_bytes for name, spec in specs.items()
|
||||
}
|
||||
assert all(cache.shape[0] == num_blocks for cache in worker.gpu_kv_caches.values())
|
||||
|
||||
# Every registered region must alias the bytes the model writes through, or
|
||||
# offloaded blocks would be copied from the wrong place.
|
||||
for name, cache in kv_caches.items():
|
||||
region = worker.gpu_kv_caches[name]
|
||||
assert region.data_ptr() == cache.data_ptr()
|
||||
assert region.stride(0) == specs[name].page_size_bytes
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rank_blocks", [1, 5])
|
||||
def test_register_mixed_page_sizes_odd_block_counts(monkeypatch, rank_blocks):
|
||||
"""Registration holds at block counts that are not powers of two.
|
||||
|
||||
``num_blocks`` comes out of ``available_memory // bytes_per_block``, so
|
||||
any integer is reachable. Registration derives regions from the config's
|
||||
own block count and strides, and the CPU pool rounds down from the same
|
||||
per-block byte total; none of it assumes anything about how the count
|
||||
was reached.
|
||||
"""
|
||||
num_layers = 4
|
||||
block_size = 64
|
||||
specs = _dsa_specs(num_layers, block_size)
|
||||
group = KVCacheGroupSpec(
|
||||
list(specs),
|
||||
UniformTypeKVCacheSpecs(block_size=block_size, kv_cache_specs=specs),
|
||||
)
|
||||
layout = KVCacheLayout.LBNHC
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.cache_config = CacheConfig()
|
||||
vllm_config.cache_config.kv_cache_layout = layout.name
|
||||
vllm_config.cache_config.num_gpu_blocks_override = None
|
||||
|
||||
pages = [spec.page_size_bytes for spec in specs.values()]
|
||||
kv_cache_config = get_kv_cache_config_from_groups(
|
||||
vllm_config, [group], sum(pages) * rank_blocks
|
||||
)
|
||||
assert kv_cache_config.num_blocks == rank_blocks
|
||||
|
||||
kv_caches = allocate_kv_cache(kv_cache_config, torch.device("cuda"), layout)
|
||||
worker = SimpleCPUOffloadWorker(
|
||||
vllm_config=None,
|
||||
kv_cache_config=kv_cache_config,
|
||||
cpu_capacity_bytes=sum(pages) * rank_blocks,
|
||||
)
|
||||
worker._backend = MagicMock()
|
||||
monkeypatch.setattr("vllm.v1.simple_kv_offload.worker.PIN_MEMORY", False)
|
||||
|
||||
worker.register_kv_caches(kv_caches)
|
||||
|
||||
assert worker.gpu_kv_caches is not None
|
||||
assert {n: c.shape for n, c in worker.gpu_kv_caches.items()} == {
|
||||
name: (rank_blocks, spec.page_size_bytes) for name, spec in specs.items()
|
||||
}
|
||||
assert worker.num_cpu_blocks == rank_blocks
|
||||
|
||||
|
||||
def test_mixed_page_byte_placement_is_dcp_invariant():
|
||||
"""DCP scales a block's token span, not its byte placement.
|
||||
|
||||
``get_kv_cache_config_from_groups`` never reads
|
||||
``decode_context_parallel_size``, so every placement field registration
|
||||
derives regions from is identical at dcp=1 and dcp=2. What DCP scales is
|
||||
the block's token span, reported by ``resolve_kv_cache_block_sizes``;
|
||||
checking that it doubles proves DCP was actually in effect, or the
|
||||
identity above would be trivially true.
|
||||
"""
|
||||
specs = _dsa_specs(4, block_size=128)
|
||||
|
||||
placements = []
|
||||
for dcp_size in (1, 2):
|
||||
group = KVCacheGroupSpec(
|
||||
list(specs),
|
||||
UniformTypeKVCacheSpecs(block_size=128, kv_cache_specs=specs),
|
||||
)
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.cache_config = CacheConfig()
|
||||
vllm_config.cache_config.block_size = 128
|
||||
vllm_config.cache_config.kv_cache_layout = KVCacheLayout.LBNHC.name
|
||||
vllm_config.cache_config.num_gpu_blocks_override = None
|
||||
vllm_config.parallel_config.decode_context_parallel_size = dcp_size
|
||||
|
||||
page_bytes = sum(spec.page_size_bytes for spec in specs.values())
|
||||
config = get_kv_cache_config_from_groups(vllm_config, [group], page_bytes * 4)
|
||||
scheduler_block_size, _ = resolve_kv_cache_block_sizes(config, vllm_config)
|
||||
assert scheduler_block_size == 128 * dcp_size
|
||||
placements.append(
|
||||
[
|
||||
(t.size, tuple(t.layers), t.layer_stride, t.block_stride, t.offset)
|
||||
for t in config.kv_cache_tensors
|
||||
]
|
||||
)
|
||||
|
||||
assert placements[0] == placements[1]
|
||||
|
||||
@@ -100,33 +100,55 @@ class SimpleCPUOffloadWorker:
|
||||
assert self.kv_cache_config is not None
|
||||
num_blocks = self.kv_cache_config.num_blocks
|
||||
assert self.kv_cache_config.kv_cache_tensors
|
||||
logical_storage_bytes = self.kv_cache_config.kv_cache_tensors[0].size
|
||||
|
||||
# The DMA backend copies whole blocks as base + block_id * stride(0),
|
||||
# so view each unique allocation as [num_blocks, block_bytes].
|
||||
# The DMA backend copies blocks as base + block_id * stride(0), so every
|
||||
# region is a [num_blocks, block_bytes] view. Block bytes come from each
|
||||
# tensor's placement -- layers in one allocation can differ in page size.
|
||||
unique_gpu_caches: dict[str, torch.Tensor] = {}
|
||||
seen: set[tuple[torch.device, int]] = set()
|
||||
for name, tensor in kv_caches.items():
|
||||
storage = tensor.untyped_storage()
|
||||
key = (tensor.device, storage.data_ptr())
|
||||
if key in seen:
|
||||
for kv_cache_tensor in self.kv_cache_config.kv_cache_tensors:
|
||||
name = next((n for n in kv_cache_tensor.layers if n in kv_caches), None)
|
||||
if name is None:
|
||||
continue
|
||||
seen.add(key)
|
||||
num_layers = len(kv_cache_tensor.layers)
|
||||
block_bytes = kv_cache_tensor.block_stride
|
||||
blocks_span = num_blocks * block_bytes
|
||||
layer_outermost = kv_cache_tensor.layer_stride >= blocks_span
|
||||
if layer_outermost:
|
||||
# One region per layer, or per KV head group under LHBNC.
|
||||
start = kv_cache_tensor.offset
|
||||
span = num_layers * kv_cache_tensor.layer_stride
|
||||
layer_names = kv_cache_tensor.layers
|
||||
else:
|
||||
start = 0
|
||||
span = blocks_span
|
||||
layer_names = kv_cache_tensor.layers[:1]
|
||||
|
||||
physical_per_block, remainder = divmod(tensor.shape[0], num_blocks)
|
||||
cache = kv_caches[name]
|
||||
raw = torch.empty(0, dtype=torch.int8, device=cache.device).set_(
|
||||
cache.untyped_storage()
|
||||
)
|
||||
assert raw.numel() >= start + span, (
|
||||
f"KV cache {name!r} storage has {raw.numel()} bytes, smaller than "
|
||||
f"the {span} bytes it places at offset {start}"
|
||||
)
|
||||
regions = raw[start : start + span].view(-1, num_blocks, block_bytes)
|
||||
groups_per_layer, remainder = divmod(len(regions), len(layer_names))
|
||||
assert remainder == 0, (
|
||||
f"KV cache {name!r} has {tensor.shape[0]} physical blocks, which "
|
||||
f"is not divisible by {num_blocks} scheduler blocks"
|
||||
f"KV cache {name!r} places {len(layer_names)} layers in "
|
||||
f"{len(regions)} block-strided regions"
|
||||
)
|
||||
block_bytes = tensor.stride(0) * tensor.element_size() * physical_per_block
|
||||
raw = torch.empty(0, dtype=torch.int8, device=tensor.device).set_(storage)
|
||||
assert raw.numel() >= logical_storage_bytes, (
|
||||
f"KV cache {name!r} storage has {raw.numel()} bytes, smaller "
|
||||
f"than the configured {logical_storage_bytes}-byte allocation"
|
||||
)
|
||||
regions = raw[:logical_storage_bytes].view(-1, num_blocks, block_bytes)
|
||||
for idx, region in enumerate(regions):
|
||||
key_name = name if len(regions) == 1 else f"{name}.{idx}"
|
||||
key = (region.device, region.data_ptr())
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
layer_name = layer_names[idx // groups_per_layer]
|
||||
key_name = (
|
||||
layer_name
|
||||
if groups_per_layer == 1
|
||||
else f"{layer_name}.{idx % groups_per_layer}"
|
||||
)
|
||||
unique_gpu_caches[key_name] = region
|
||||
|
||||
# Compute per-tensor bytes_per_block. Tensors may have different
|
||||
|
||||
Reference in New Issue
Block a user