[Bugfix][KV Offload] Handle padded GPU cache storage (#54021)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
6c4aa8bec3
commit
478ec3ea41
@@ -209,7 +209,10 @@ def test_register_shared_kv_cache_storage(monkeypatch, layout: KVCacheLayout):
|
||||
device="cuda",
|
||||
)
|
||||
caches = dense_kv_cache_views(raw, spec, num_blocks, num_layers, layout)
|
||||
cache_config = MagicMock(num_blocks=num_blocks)
|
||||
cache_config = MagicMock(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[MagicMock(size=raw.nbytes)],
|
||||
)
|
||||
worker = SimpleCPUOffloadWorker(
|
||||
vllm_config=None,
|
||||
kv_cache_config=cache_config,
|
||||
@@ -238,6 +241,30 @@ def test_register_shared_kv_cache_storage(monkeypatch, layout: KVCacheLayout):
|
||||
}
|
||||
|
||||
|
||||
def test_register_kv_cache_storage_with_trailing_padding(monkeypatch):
|
||||
num_blocks = 4
|
||||
block_bytes = 32
|
||||
cache_bytes = num_blocks * block_bytes
|
||||
raw = torch.zeros(4096, dtype=torch.int8, device="cuda")
|
||||
cache = raw[:cache_bytes].view(num_blocks, block_bytes)
|
||||
worker = SimpleCPUOffloadWorker(
|
||||
vllm_config=None,
|
||||
kv_cache_config=MagicMock(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[MagicMock(size=cache_bytes)],
|
||||
),
|
||||
cpu_capacity_bytes=cache_bytes,
|
||||
)
|
||||
worker._backend = MagicMock()
|
||||
monkeypatch.setattr("vllm.v1.simple_kv_offload.worker.PIN_MEMORY", False)
|
||||
|
||||
worker.register_kv_caches({"layer.0": cache})
|
||||
|
||||
assert worker.gpu_kv_caches is not None
|
||||
assert list(worker.gpu_kv_caches) == ["layer.0"]
|
||||
assert worker.gpu_kv_caches["layer.0"].shape == (num_blocks, block_bytes)
|
||||
|
||||
|
||||
def test_register_separate_kv_head_groups(monkeypatch):
|
||||
# LHBNC hoists the K/V head groups outside the block dim, so each layer's
|
||||
# blocks are registered as one region per group (K, V).
|
||||
@@ -260,7 +287,10 @@ def test_register_separate_kv_head_groups(monkeypatch):
|
||||
caches = dense_kv_cache_views(raw, spec, num_blocks, num_layers, layout)
|
||||
worker = SimpleCPUOffloadWorker(
|
||||
vllm_config=None,
|
||||
kv_cache_config=MagicMock(num_blocks=num_blocks),
|
||||
kv_cache_config=MagicMock(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[MagicMock(size=raw.nbytes)],
|
||||
),
|
||||
cpu_capacity_bytes=raw.nbytes,
|
||||
)
|
||||
worker._backend = MagicMock()
|
||||
|
||||
@@ -99,6 +99,8 @@ class SimpleCPUOffloadWorker:
|
||||
|
||||
assert self.kv_cache_config is not None
|
||||
num_blocks = self.kv_cache_config.num_blocks
|
||||
assert self.kv_cache_config.kv_cache_tensors
|
||||
logical_storage_bytes = self.kv_cache_config.kv_cache_tensors[0].size
|
||||
|
||||
# The DMA backend copies whole blocks as base + block_id * stride(0),
|
||||
# so view each unique allocation as [num_blocks, block_bytes].
|
||||
@@ -118,7 +120,11 @@ class SimpleCPUOffloadWorker:
|
||||
)
|
||||
block_bytes = tensor.stride(0) * tensor.element_size() * physical_per_block
|
||||
raw = torch.empty(0, dtype=torch.int8, device=tensor.device).set_(storage)
|
||||
regions = raw.view(-1, num_blocks, block_bytes)
|
||||
assert raw.numel() >= logical_storage_bytes, (
|
||||
f"KV cache {name!r} storage has {raw.numel()} bytes, smaller "
|
||||
f"than the configured {logical_storage_bytes}-byte allocation"
|
||||
)
|
||||
regions = raw[:logical_storage_bytes].view(-1, num_blocks, block_bytes)
|
||||
for idx, region in enumerate(regions):
|
||||
key_name = name if len(regions) == 1 else f"{name}.{idx}"
|
||||
unique_gpu_caches[key_name] = region
|
||||
|
||||
Reference in New Issue
Block a user