[Bugfix][KV Offload] Handle padded GPU cache storage (#54021)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Jiangyun Zhu
2026-08-27 10:52:59 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 6c4aa8bec3
commit 478ec3ea41
2 changed files with 39 additions and 3 deletions
+32 -2
View File
@@ -209,7 +209,10 @@ def test_register_shared_kv_cache_storage(monkeypatch, layout: KVCacheLayout):
device="cuda",
)
caches = dense_kv_cache_views(raw, spec, num_blocks, num_layers, layout)
cache_config = MagicMock(num_blocks=num_blocks)
cache_config = MagicMock(
num_blocks=num_blocks,
kv_cache_tensors=[MagicMock(size=raw.nbytes)],
)
worker = SimpleCPUOffloadWorker(
vllm_config=None,
kv_cache_config=cache_config,
@@ -238,6 +241,30 @@ def test_register_shared_kv_cache_storage(monkeypatch, layout: KVCacheLayout):
}
def test_register_kv_cache_storage_with_trailing_padding(monkeypatch):
num_blocks = 4
block_bytes = 32
cache_bytes = num_blocks * block_bytes
raw = torch.zeros(4096, dtype=torch.int8, device="cuda")
cache = raw[:cache_bytes].view(num_blocks, block_bytes)
worker = SimpleCPUOffloadWorker(
vllm_config=None,
kv_cache_config=MagicMock(
num_blocks=num_blocks,
kv_cache_tensors=[MagicMock(size=cache_bytes)],
),
cpu_capacity_bytes=cache_bytes,
)
worker._backend = MagicMock()
monkeypatch.setattr("vllm.v1.simple_kv_offload.worker.PIN_MEMORY", False)
worker.register_kv_caches({"layer.0": cache})
assert worker.gpu_kv_caches is not None
assert list(worker.gpu_kv_caches) == ["layer.0"]
assert worker.gpu_kv_caches["layer.0"].shape == (num_blocks, block_bytes)
def test_register_separate_kv_head_groups(monkeypatch):
# LHBNC hoists the K/V head groups outside the block dim, so each layer's
# blocks are registered as one region per group (K, V).
@@ -260,7 +287,10 @@ def test_register_separate_kv_head_groups(monkeypatch):
caches = dense_kv_cache_views(raw, spec, num_blocks, num_layers, layout)
worker = SimpleCPUOffloadWorker(
vllm_config=None,
kv_cache_config=MagicMock(num_blocks=num_blocks),
kv_cache_config=MagicMock(
num_blocks=num_blocks,
kv_cache_tensors=[MagicMock(size=raw.nbytes)],
),
cpu_capacity_bytes=raw.nbytes,
)
worker._backend = MagicMock()
+7 -1
View File
@@ -99,6 +99,8 @@ class SimpleCPUOffloadWorker:
assert self.kv_cache_config is not None
num_blocks = self.kv_cache_config.num_blocks
assert self.kv_cache_config.kv_cache_tensors
logical_storage_bytes = self.kv_cache_config.kv_cache_tensors[0].size
# The DMA backend copies whole blocks as base + block_id * stride(0),
# so view each unique allocation as [num_blocks, block_bytes].
@@ -118,7 +120,11 @@ class SimpleCPUOffloadWorker:
)
block_bytes = tensor.stride(0) * tensor.element_size() * physical_per_block
raw = torch.empty(0, dtype=torch.int8, device=tensor.device).set_(storage)
regions = raw.view(-1, num_blocks, block_bytes)
assert raw.numel() >= logical_storage_bytes, (
f"KV cache {name!r} storage has {raw.numel()} bytes, smaller "
f"than the configured {logical_storage_bytes}-byte allocation"
)
regions = raw[:logical_storage_bytes].view(-1, num_blocks, block_bytes)
for idx, region in enumerate(regions):
key_name = name if len(regions) == 1 else f"{name}.{idx}"
unique_gpu_caches[key_name] = region