[Bugfix][KV Offload] Handle padded GPU cache storage (#54021)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
6c4aa8bec3
commit
478ec3ea41
@@ -209,7 +209,10 @@ def test_register_shared_kv_cache_storage(monkeypatch, layout: KVCacheLayout):
|
||||
device="cuda",
|
||||
)
|
||||
caches = dense_kv_cache_views(raw, spec, num_blocks, num_layers, layout)
|
||||
cache_config = MagicMock(num_blocks=num_blocks)
|
||||
cache_config = MagicMock(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[MagicMock(size=raw.nbytes)],
|
||||
)
|
||||
worker = SimpleCPUOffloadWorker(
|
||||
vllm_config=None,
|
||||
kv_cache_config=cache_config,
|
||||
@@ -238,6 +241,30 @@ def test_register_shared_kv_cache_storage(monkeypatch, layout: KVCacheLayout):
|
||||
}
|
||||
|
||||
|
||||
def test_register_kv_cache_storage_with_trailing_padding(monkeypatch):
|
||||
num_blocks = 4
|
||||
block_bytes = 32
|
||||
cache_bytes = num_blocks * block_bytes
|
||||
raw = torch.zeros(4096, dtype=torch.int8, device="cuda")
|
||||
cache = raw[:cache_bytes].view(num_blocks, block_bytes)
|
||||
worker = SimpleCPUOffloadWorker(
|
||||
vllm_config=None,
|
||||
kv_cache_config=MagicMock(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[MagicMock(size=cache_bytes)],
|
||||
),
|
||||
cpu_capacity_bytes=cache_bytes,
|
||||
)
|
||||
worker._backend = MagicMock()
|
||||
monkeypatch.setattr("vllm.v1.simple_kv_offload.worker.PIN_MEMORY", False)
|
||||
|
||||
worker.register_kv_caches({"layer.0": cache})
|
||||
|
||||
assert worker.gpu_kv_caches is not None
|
||||
assert list(worker.gpu_kv_caches) == ["layer.0"]
|
||||
assert worker.gpu_kv_caches["layer.0"].shape == (num_blocks, block_bytes)
|
||||
|
||||
|
||||
def test_register_separate_kv_head_groups(monkeypatch):
|
||||
# LHBNC hoists the K/V head groups outside the block dim, so each layer's
|
||||
# blocks are registered as one region per group (K, V).
|
||||
@@ -260,7 +287,10 @@ def test_register_separate_kv_head_groups(monkeypatch):
|
||||
caches = dense_kv_cache_views(raw, spec, num_blocks, num_layers, layout)
|
||||
worker = SimpleCPUOffloadWorker(
|
||||
vllm_config=None,
|
||||
kv_cache_config=MagicMock(num_blocks=num_blocks),
|
||||
kv_cache_config=MagicMock(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[MagicMock(size=raw.nbytes)],
|
||||
),
|
||||
cpu_capacity_bytes=raw.nbytes,
|
||||
)
|
||||
worker._backend = MagicMock()
|
||||
|
||||
Reference in New Issue
Block a user