[KV Offload] Support packed HMA KV cache layout (#46205)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Tyler Michael Smith
parent
77148992cf
commit
cc22621b51
@@ -1,16 +1,23 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for contiguous KV cache packing in _get_kv_cache_config_deepseek_v4."""
|
||||
"""Tests for contiguous KV cache packing."""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.core.kv_cache_utils import _get_kv_cache_config_deepseek_v4
|
||||
from vllm import envs
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
_get_kv_cache_config_deepseek_v4,
|
||||
get_kv_cache_config_from_groups,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
|
||||
@@ -28,6 +35,25 @@ def _make_mla_spec(page_size: int, block_size: int = 256) -> MLAAttentionSpec:
|
||||
)
|
||||
|
||||
|
||||
def _make_full_spec() -> FullAttentionSpec:
|
||||
return FullAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=2,
|
||||
head_size=64,
|
||||
dtype=torch.float16,
|
||||
)
|
||||
|
||||
|
||||
def _make_sw_spec() -> SlidingWindowSpec:
|
||||
return SlidingWindowSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=2,
|
||||
head_size=64,
|
||||
dtype=torch.float16,
|
||||
sliding_window=128,
|
||||
)
|
||||
|
||||
|
||||
def _make_groups(n_c4, n_c128, n_swa):
|
||||
PS_C4_MLA = 37440
|
||||
PS_C4_IDX = 8640
|
||||
@@ -130,6 +156,73 @@ class TestInterleavedPacking:
|
||||
for i, v in enumerate(views):
|
||||
assert (v == i + 1).all(), f"View {i} was corrupted"
|
||||
|
||||
def test_hma_attention_groups_keep_default_backing(self, monkeypatch):
|
||||
monkeypatch.setattr(envs, "VLLM_USE_PACKED_HMA_KV_CACHE", False, raising=False)
|
||||
full = _make_full_spec()
|
||||
sw = _make_sw_spec()
|
||||
page_size = full.page_size_bytes
|
||||
groups = [
|
||||
KVCacheGroupSpec(["full.0", "full.1"], full),
|
||||
KVCacheGroupSpec(["sw.0", "sw.2"], sw),
|
||||
KVCacheGroupSpec(["sw.1", "sw.3"], sw),
|
||||
]
|
||||
|
||||
config = get_kv_cache_config_from_groups(
|
||||
_mock_vllm_config(), groups, available_memory=page_size * 2 * 32
|
||||
)
|
||||
|
||||
assert config.num_blocks == 32
|
||||
assert sum(t.size for t in config.kv_cache_tensors) == page_size * 2 * 32
|
||||
assert config.kv_cache_tensors == [
|
||||
KVCacheTensor(size=page_size * 32, shared_by=["full.0", "sw.0", "sw.1"]),
|
||||
KVCacheTensor(size=page_size * 32, shared_by=["full.1", "sw.2", "sw.3"]),
|
||||
]
|
||||
|
||||
def test_hma_attention_groups_use_packed_backing_with_flag(self, monkeypatch):
|
||||
monkeypatch.setattr(envs, "VLLM_USE_PACKED_HMA_KV_CACHE", True, raising=False)
|
||||
full = _make_full_spec()
|
||||
sw = _make_sw_spec()
|
||||
page_size = full.page_size_bytes
|
||||
groups = [
|
||||
KVCacheGroupSpec(["full.0", "full.1"], full),
|
||||
KVCacheGroupSpec(["sw.0", "sw.2"], sw),
|
||||
KVCacheGroupSpec(["sw.1", "sw.3"], sw),
|
||||
]
|
||||
|
||||
config = get_kv_cache_config_from_groups(
|
||||
_mock_vllm_config(), groups, available_memory=page_size * 2 * 32
|
||||
)
|
||||
|
||||
assert config.num_blocks == 32
|
||||
assert {t.size for t in config.kv_cache_tensors} == {page_size * 2 * 32}
|
||||
assert config.kv_cache_tensors == [
|
||||
KVCacheTensor(
|
||||
size=page_size * 2 * 32,
|
||||
shared_by=["full.0", "sw.0", "sw.1"],
|
||||
offset=0,
|
||||
block_stride=page_size * 2,
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=page_size * 2 * 32,
|
||||
shared_by=["full.1", "sw.2", "sw.3"],
|
||||
offset=page_size,
|
||||
block_stride=page_size * 2,
|
||||
),
|
||||
]
|
||||
|
||||
def test_single_group_attention_keeps_unpacked_layout(self):
|
||||
spec = _make_full_spec()
|
||||
groups = [KVCacheGroupSpec(["full.0", "full.1"], spec)]
|
||||
|
||||
config = get_kv_cache_config_from_groups(
|
||||
_mock_vllm_config(), groups, available_memory=spec.page_size_bytes * 2 * 32
|
||||
)
|
||||
|
||||
assert sum(t.size for t in config.kv_cache_tensors) == (
|
||||
spec.page_size_bytes * 2 * 32
|
||||
)
|
||||
assert [t.block_stride for t in config.kv_cache_tensors] == [0, 0]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
|
||||
Reference in New Issue
Block a user