From 478ec3ea41358d410a0997d8852ef5a344c9a677 Mon Sep 17 00:00:00 2001 From: Jiangyun Zhu Date: Thu, 27 Aug 2026 18:52:59 +0800 Subject: [PATCH] [Bugfix][KV Offload] Handle padded GPU cache storage (#54021) Signed-off-by: zjy0516 Co-authored-by: OpenAI Codex --- tests/v1/simple_kv_offload/test_worker.py | 34 +++++++++++++++++++++-- vllm/v1/simple_kv_offload/worker.py | 8 +++++- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/tests/v1/simple_kv_offload/test_worker.py b/tests/v1/simple_kv_offload/test_worker.py index 645cd5daba..ccc47dc387 100644 --- a/tests/v1/simple_kv_offload/test_worker.py +++ b/tests/v1/simple_kv_offload/test_worker.py @@ -209,7 +209,10 @@ def test_register_shared_kv_cache_storage(monkeypatch, layout: KVCacheLayout): device="cuda", ) caches = dense_kv_cache_views(raw, spec, num_blocks, num_layers, layout) - cache_config = MagicMock(num_blocks=num_blocks) + cache_config = MagicMock( + num_blocks=num_blocks, + kv_cache_tensors=[MagicMock(size=raw.nbytes)], + ) worker = SimpleCPUOffloadWorker( vllm_config=None, kv_cache_config=cache_config, @@ -238,6 +241,30 @@ def test_register_shared_kv_cache_storage(monkeypatch, layout: KVCacheLayout): } +def test_register_kv_cache_storage_with_trailing_padding(monkeypatch): + num_blocks = 4 + block_bytes = 32 + cache_bytes = num_blocks * block_bytes + raw = torch.zeros(4096, dtype=torch.int8, device="cuda") + cache = raw[:cache_bytes].view(num_blocks, block_bytes) + worker = SimpleCPUOffloadWorker( + vllm_config=None, + kv_cache_config=MagicMock( + num_blocks=num_blocks, + kv_cache_tensors=[MagicMock(size=cache_bytes)], + ), + cpu_capacity_bytes=cache_bytes, + ) + worker._backend = MagicMock() + monkeypatch.setattr("vllm.v1.simple_kv_offload.worker.PIN_MEMORY", False) + + worker.register_kv_caches({"layer.0": cache}) + + assert worker.gpu_kv_caches is not None + assert list(worker.gpu_kv_caches) == ["layer.0"] + assert worker.gpu_kv_caches["layer.0"].shape == (num_blocks, block_bytes) + + def test_register_separate_kv_head_groups(monkeypatch): # LHBNC hoists the K/V head groups outside the block dim, so each layer's # blocks are registered as one region per group (K, V). @@ -260,7 +287,10 @@ def test_register_separate_kv_head_groups(monkeypatch): caches = dense_kv_cache_views(raw, spec, num_blocks, num_layers, layout) worker = SimpleCPUOffloadWorker( vllm_config=None, - kv_cache_config=MagicMock(num_blocks=num_blocks), + kv_cache_config=MagicMock( + num_blocks=num_blocks, + kv_cache_tensors=[MagicMock(size=raw.nbytes)], + ), cpu_capacity_bytes=raw.nbytes, ) worker._backend = MagicMock() diff --git a/vllm/v1/simple_kv_offload/worker.py b/vllm/v1/simple_kv_offload/worker.py index f2925915d8..dc95d4d3e2 100644 --- a/vllm/v1/simple_kv_offload/worker.py +++ b/vllm/v1/simple_kv_offload/worker.py @@ -99,6 +99,8 @@ class SimpleCPUOffloadWorker: assert self.kv_cache_config is not None num_blocks = self.kv_cache_config.num_blocks + assert self.kv_cache_config.kv_cache_tensors + logical_storage_bytes = self.kv_cache_config.kv_cache_tensors[0].size # The DMA backend copies whole blocks as base + block_id * stride(0), # so view each unique allocation as [num_blocks, block_bytes]. @@ -118,7 +120,11 @@ class SimpleCPUOffloadWorker: ) block_bytes = tensor.stride(0) * tensor.element_size() * physical_per_block raw = torch.empty(0, dtype=torch.int8, device=tensor.device).set_(storage) - regions = raw.view(-1, num_blocks, block_bytes) + assert raw.numel() >= logical_storage_bytes, ( + f"KV cache {name!r} storage has {raw.numel()} bytes, smaller " + f"than the configured {logical_storage_bytes}-byte allocation" + ) + regions = raw[:logical_storage_bytes].view(-1, num_blocks, block_bytes) for idx, region in enumerate(regions): key_name = name if len(regions) == 1 else f"{name}.{idx}" unique_gpu_caches[key_name] = region