From eb6b619ab256df1100cb75684a9ba78485506c3a Mon Sep 17 00:00:00 2001 From: He Liu Date: Tue, 8 Sep 2026 11:50:15 +0800 Subject: [PATCH] [Bugfix][KV Offload] Register mixed page sizes in one cache group (#54756) Signed-off-by: heliubj18 Co-authored-by: Kimi --- tests/v1/attention/utils.py | 30 ++- tests/v1/simple_kv_offload/test_worker.py | 220 +++++++++++++++++++++- vllm/v1/simple_kv_offload/worker.py | 60 ++++-- 3 files changed, 277 insertions(+), 33 deletions(-) diff --git a/tests/v1/attention/utils.py b/tests/v1/attention/utils.py index ee702bdbdf..590c9efb8a 100644 --- a/tests/v1/attention/utils.py +++ b/tests/v1/attention/utils.py @@ -426,6 +426,26 @@ class MockMambaBuilder(BaseMambaAttentionMetadataBuilder[BaseMambaAttentionMetad return builder.build(0, common_metadata) +def dense_kv_cache_tensor( + raw: torch.Tensor, + spec: KVCacheSpec, + num_blocks: int, + num_layers: int, + layout: KVCacheLayout, + layer_names: list[str] | None = None, +) -> KVCacheTensor: + """The ``KVCacheTensor`` for a dense allocation of ``num_layers`` layers.""" + layer_stride, block_stride, _, _, _ = compute_layout_strides( + spec, num_blocks, num_layers, layout + ) + return KVCacheTensor( + size=raw.numel(), + layers=layer_names or [str(i) for i in range(num_layers)], + layer_stride=layer_stride, + block_stride=block_stride, + ) + + def dense_kv_cache_views( raw: torch.Tensor, spec: KVCacheSpec, @@ -435,15 +455,7 @@ def dense_kv_cache_views( kernel_block_size: int | None = None, ) -> list[torch.Tensor]: """``create_kv_cache_views`` for a dense allocation of ``num_layers`` layers.""" - layer_stride, block_stride, _, _, _ = compute_layout_strides( - spec, num_blocks, num_layers, layout - ) - tensor = KVCacheTensor( - size=raw.numel(), - layers=[str(i) for i in range(num_layers)], - layer_stride=layer_stride, - block_stride=block_stride, - ) + tensor = dense_kv_cache_tensor(raw, spec, num_blocks, num_layers, layout) return create_kv_cache_views( raw, spec, diff --git a/tests/v1/simple_kv_offload/test_worker.py b/tests/v1/simple_kv_offload/test_worker.py index 473c3983ad..66582f03bb 100644 --- a/tests/v1/simple_kv_offload/test_worker.py +++ b/tests/v1/simple_kv_offload/test_worker.py @@ -20,8 +20,22 @@ from vllm.platforms import current_platform if not current_platform.is_cuda_alike(): pytest.skip("Requires CUDA or ROCm", allow_module_level=True) -from tests.v1.attention.utils import dense_kv_cache_views -from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheLayout +from tests.v1.attention.utils import dense_kv_cache_tensor, dense_kv_cache_views +from vllm.config import CacheConfig +from vllm.v1.core.kv_cache_utils import ( + get_kv_cache_config_from_groups, + is_kv_cache_spec_uniform, + resolve_kv_cache_block_sizes, +) +from vllm.v1.kv_cache_interface import ( + FullAttentionSpec, + KVCacheGroupSpec, + KVCacheLayout, + KVCacheSpec, + KVCacheTensor, + MLAAttentionSpec, + UniformTypeKVCacheSpecs, +) from vllm.v1.simple_kv_offload.copy_backend import DmaCopyBackend from vllm.v1.simple_kv_offload.cuda_mem_ops import ( CU_MEMCPY_SRC_ACCESS_ORDER_ANY, @@ -31,6 +45,7 @@ from vllm.v1.simple_kv_offload.cuda_mem_ops import ( ) from vllm.v1.simple_kv_offload.metadata import SimpleCPUOffloadMetadata from vllm.v1.simple_kv_offload.worker import SimpleCPUOffloadWorker +from vllm.v1.worker.utils import allocate_kv_cache NUM_BLOCKS = 64 BLOCK_BYTES = 4096 @@ -210,9 +225,14 @@ def test_register_shared_kv_cache_storage(monkeypatch, layout: KVCacheLayout): device="cuda", ) caches = dense_kv_cache_views(raw, spec, num_blocks, num_layers, layout) + layer_names = [f"layer.{i}" for i in range(num_layers)] cache_config = MagicMock( num_blocks=num_blocks, - kv_cache_tensors=[MagicMock(size=raw.nbytes)], + kv_cache_tensors=[ + dense_kv_cache_tensor( + raw, spec, num_blocks, num_layers, layout, layer_names + ) + ], ) worker = SimpleCPUOffloadWorker( vllm_config=None, @@ -252,7 +272,14 @@ def test_register_kv_cache_storage_with_trailing_padding(monkeypatch): vllm_config=None, kv_cache_config=MagicMock( num_blocks=num_blocks, - kv_cache_tensors=[MagicMock(size=cache_bytes)], + kv_cache_tensors=[ + KVCacheTensor( + size=cache_bytes, + layers=["layer.0"], + layer_stride=cache_bytes, + block_stride=block_bytes, + ) + ], ), cpu_capacity_bytes=cache_bytes, ) @@ -286,11 +313,16 @@ def test_register_separate_kv_head_groups(monkeypatch): device="cuda", ) caches = dense_kv_cache_views(raw, spec, num_blocks, num_layers, layout) + layer_names = [f"layer.{i}" for i in range(num_layers)] worker = SimpleCPUOffloadWorker( vllm_config=None, kv_cache_config=MagicMock( num_blocks=num_blocks, - kv_cache_tensors=[MagicMock(size=raw.nbytes)], + kv_cache_tensors=[ + dense_kv_cache_tensor( + raw, spec, num_blocks, num_layers, layout, layer_names + ) + ], ), cpu_capacity_bytes=raw.nbytes, ) @@ -309,3 +341,181 @@ def test_register_separate_kv_head_groups(monkeypatch): assert {cache.shape for cache in worker.gpu_kv_caches.values()} == { (num_blocks, per_group_block_bytes) } + + +def _dsa_specs(num_layers: int, block_size: int) -> dict[str, KVCacheSpec]: + """A DSA model's per-layer specs (DeepSeek-V3.2, GLM-5.2). + + Each decoder layer contributes an MLA latent cache and the indexer's key + cache, whose pages differ in size (``MLAAttentionSpec`` off ``head_size``: + 512 + 64 for the latent, 128 fp8 keys + one fp32 scale per 128 elements for + the indexer). Both are ``MLAAttentionSpec`` with the same block size, so the + hybrid allocator puts them in one ``UniformTypeKVCacheSpecs`` cache group. + See ``DeepseekV32IndexerCache.get_kv_cache_spec`` and + ``MLAAttention.get_kv_cache_spec``. + """ + specs: dict[str, KVCacheSpec] = {} + for i in range(num_layers): + specs[f"model.layers.{i}.self_attn.attn"] = MLAAttentionSpec( + block_size=block_size, + num_kv_heads=1, + head_size=512 + 64, + dtype=torch.uint8, + cache_dtype_str="fp8", + ) + specs[f"model.layers.{i}.self_attn.indexer.k_cache"] = MLAAttentionSpec( + block_size=block_size, + num_kv_heads=1, + head_size=128 + 128 // 128 * 4, + dtype=torch.uint8, + ) + return specs + + +def test_register_mixed_page_sizes_in_one_cache_group(monkeypatch): + """Sparse-MLA models mix page sizes inside one cache group. + + The MLA latent and indexer key caches of a layer have different page sizes, + so the allocation has no single per-layer block size to reinterpret it with: + ``num_layers * (mla_page + indexer_page)`` is not a multiple of either page. + Registration must derive each region from the placement metadata instead. + """ + num_layers = 4 + block_size = 64 + specs = _dsa_specs(num_layers, block_size) + # Differing head sizes make the specs non-identical but same-type, which is + # what lands both caches of a layer in one group. + assert not is_kv_cache_spec_uniform(specs) + assert UniformTypeKVCacheSpecs.is_uniform_type(specs) + group = KVCacheGroupSpec( + list(specs), + UniformTypeKVCacheSpecs(block_size=block_size, kv_cache_specs=specs), + ) + + # DEEPSEEK_V32_INDEXER declares no supported layouts, so resolution lands on + # the default preference: layer-outermost, one region per layer. + layout = KVCacheLayout.LBNHC + vllm_config = MagicMock() + vllm_config.cache_config = CacheConfig() + vllm_config.cache_config.kv_cache_layout = layout.name + vllm_config.cache_config.num_gpu_blocks_override = None + + pages = [spec.page_size_bytes for spec in specs.values()] + num_blocks = 4 + kv_cache_config = get_kv_cache_config_from_groups( + vllm_config, [group], sum(pages) * num_blocks + ) + assert kv_cache_config.num_blocks == num_blocks + assert len(set(pages)) > 1, "the mixed page sizes are what this test covers" + + kv_caches = allocate_kv_cache(kv_cache_config, torch.device("cuda"), layout) + worker = SimpleCPUOffloadWorker( + vllm_config=None, + kv_cache_config=kv_cache_config, + cpu_capacity_bytes=sum(pages) * num_blocks, + ) + worker._backend = MagicMock() + monkeypatch.setattr("vllm.v1.simple_kv_offload.worker.PIN_MEMORY", False) + + worker.register_kv_caches(kv_caches) + + assert worker.gpu_kv_caches is not None + # One region per layer, each with that layer's own page as its block stride. + assert sorted(worker.gpu_kv_caches) == sorted(specs) + assert {name: cache.shape[1] for name, cache in worker.gpu_kv_caches.items()} == { + name: spec.page_size_bytes for name, spec in specs.items() + } + assert all(cache.shape[0] == num_blocks for cache in worker.gpu_kv_caches.values()) + + # Every registered region must alias the bytes the model writes through, or + # offloaded blocks would be copied from the wrong place. + for name, cache in kv_caches.items(): + region = worker.gpu_kv_caches[name] + assert region.data_ptr() == cache.data_ptr() + assert region.stride(0) == specs[name].page_size_bytes + + +@pytest.mark.parametrize("rank_blocks", [1, 5]) +def test_register_mixed_page_sizes_odd_block_counts(monkeypatch, rank_blocks): + """Registration holds at block counts that are not powers of two. + + ``num_blocks`` comes out of ``available_memory // bytes_per_block``, so + any integer is reachable. Registration derives regions from the config's + own block count and strides, and the CPU pool rounds down from the same + per-block byte total; none of it assumes anything about how the count + was reached. + """ + num_layers = 4 + block_size = 64 + specs = _dsa_specs(num_layers, block_size) + group = KVCacheGroupSpec( + list(specs), + UniformTypeKVCacheSpecs(block_size=block_size, kv_cache_specs=specs), + ) + layout = KVCacheLayout.LBNHC + vllm_config = MagicMock() + vllm_config.cache_config = CacheConfig() + vllm_config.cache_config.kv_cache_layout = layout.name + vllm_config.cache_config.num_gpu_blocks_override = None + + pages = [spec.page_size_bytes for spec in specs.values()] + kv_cache_config = get_kv_cache_config_from_groups( + vllm_config, [group], sum(pages) * rank_blocks + ) + assert kv_cache_config.num_blocks == rank_blocks + + kv_caches = allocate_kv_cache(kv_cache_config, torch.device("cuda"), layout) + worker = SimpleCPUOffloadWorker( + vllm_config=None, + kv_cache_config=kv_cache_config, + cpu_capacity_bytes=sum(pages) * rank_blocks, + ) + worker._backend = MagicMock() + monkeypatch.setattr("vllm.v1.simple_kv_offload.worker.PIN_MEMORY", False) + + worker.register_kv_caches(kv_caches) + + assert worker.gpu_kv_caches is not None + assert {n: c.shape for n, c in worker.gpu_kv_caches.items()} == { + name: (rank_blocks, spec.page_size_bytes) for name, spec in specs.items() + } + assert worker.num_cpu_blocks == rank_blocks + + +def test_mixed_page_byte_placement_is_dcp_invariant(): + """DCP scales a block's token span, not its byte placement. + + ``get_kv_cache_config_from_groups`` never reads + ``decode_context_parallel_size``, so every placement field registration + derives regions from is identical at dcp=1 and dcp=2. What DCP scales is + the block's token span, reported by ``resolve_kv_cache_block_sizes``; + checking that it doubles proves DCP was actually in effect, or the + identity above would be trivially true. + """ + specs = _dsa_specs(4, block_size=128) + + placements = [] + for dcp_size in (1, 2): + group = KVCacheGroupSpec( + list(specs), + UniformTypeKVCacheSpecs(block_size=128, kv_cache_specs=specs), + ) + vllm_config = MagicMock() + vllm_config.cache_config = CacheConfig() + vllm_config.cache_config.block_size = 128 + vllm_config.cache_config.kv_cache_layout = KVCacheLayout.LBNHC.name + vllm_config.cache_config.num_gpu_blocks_override = None + vllm_config.parallel_config.decode_context_parallel_size = dcp_size + + page_bytes = sum(spec.page_size_bytes for spec in specs.values()) + config = get_kv_cache_config_from_groups(vllm_config, [group], page_bytes * 4) + scheduler_block_size, _ = resolve_kv_cache_block_sizes(config, vllm_config) + assert scheduler_block_size == 128 * dcp_size + placements.append( + [ + (t.size, tuple(t.layers), t.layer_stride, t.block_stride, t.offset) + for t in config.kv_cache_tensors + ] + ) + + assert placements[0] == placements[1] diff --git a/vllm/v1/simple_kv_offload/worker.py b/vllm/v1/simple_kv_offload/worker.py index d49de261e5..9efc544f2f 100644 --- a/vllm/v1/simple_kv_offload/worker.py +++ b/vllm/v1/simple_kv_offload/worker.py @@ -100,33 +100,55 @@ class SimpleCPUOffloadWorker: assert self.kv_cache_config is not None num_blocks = self.kv_cache_config.num_blocks assert self.kv_cache_config.kv_cache_tensors - logical_storage_bytes = self.kv_cache_config.kv_cache_tensors[0].size - # The DMA backend copies whole blocks as base + block_id * stride(0), - # so view each unique allocation as [num_blocks, block_bytes]. + # The DMA backend copies blocks as base + block_id * stride(0), so every + # region is a [num_blocks, block_bytes] view. Block bytes come from each + # tensor's placement -- layers in one allocation can differ in page size. unique_gpu_caches: dict[str, torch.Tensor] = {} seen: set[tuple[torch.device, int]] = set() - for name, tensor in kv_caches.items(): - storage = tensor.untyped_storage() - key = (tensor.device, storage.data_ptr()) - if key in seen: + for kv_cache_tensor in self.kv_cache_config.kv_cache_tensors: + name = next((n for n in kv_cache_tensor.layers if n in kv_caches), None) + if name is None: continue - seen.add(key) + num_layers = len(kv_cache_tensor.layers) + block_bytes = kv_cache_tensor.block_stride + blocks_span = num_blocks * block_bytes + layer_outermost = kv_cache_tensor.layer_stride >= blocks_span + if layer_outermost: + # One region per layer, or per KV head group under LHBNC. + start = kv_cache_tensor.offset + span = num_layers * kv_cache_tensor.layer_stride + layer_names = kv_cache_tensor.layers + else: + start = 0 + span = blocks_span + layer_names = kv_cache_tensor.layers[:1] - physical_per_block, remainder = divmod(tensor.shape[0], num_blocks) + cache = kv_caches[name] + raw = torch.empty(0, dtype=torch.int8, device=cache.device).set_( + cache.untyped_storage() + ) + assert raw.numel() >= start + span, ( + f"KV cache {name!r} storage has {raw.numel()} bytes, smaller than " + f"the {span} bytes it places at offset {start}" + ) + regions = raw[start : start + span].view(-1, num_blocks, block_bytes) + groups_per_layer, remainder = divmod(len(regions), len(layer_names)) assert remainder == 0, ( - f"KV cache {name!r} has {tensor.shape[0]} physical blocks, which " - f"is not divisible by {num_blocks} scheduler blocks" + f"KV cache {name!r} places {len(layer_names)} layers in " + f"{len(regions)} block-strided regions" ) - block_bytes = tensor.stride(0) * tensor.element_size() * physical_per_block - raw = torch.empty(0, dtype=torch.int8, device=tensor.device).set_(storage) - assert raw.numel() >= logical_storage_bytes, ( - f"KV cache {name!r} storage has {raw.numel()} bytes, smaller " - f"than the configured {logical_storage_bytes}-byte allocation" - ) - regions = raw[:logical_storage_bytes].view(-1, num_blocks, block_bytes) for idx, region in enumerate(regions): - key_name = name if len(regions) == 1 else f"{name}.{idx}" + key = (region.device, region.data_ptr()) + if key in seen: + continue + seen.add(key) + layer_name = layer_names[idx // groups_per_layer] + key_name = ( + layer_name + if groups_per_layer == 1 + else f"{layer_name}.{idx % groups_per_layer}" + ) unique_gpu_caches[key_name] = region # Compute per-tensor bytes_per_block. Tensors may have different