[KV Offload] Gate packed HMA KV cache on cross-layer config (#46252)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
@@ -7,9 +7,8 @@ from unittest.mock import MagicMock
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
_get_kv_cache_config_deepseek_v4,
|
||||
_get_kv_cache_config_packed,
|
||||
get_kv_cache_config_from_groups,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
@@ -84,15 +83,19 @@ def _make_groups(n_c4, n_c128, n_swa):
|
||||
return [mla_group, swa_group]
|
||||
|
||||
|
||||
def _mock_vllm_config():
|
||||
def _mock_vllm_config(kv_connector_extra_config: dict[str, str] | None = None):
|
||||
config = MagicMock()
|
||||
config.cache_config.num_gpu_blocks_override = None
|
||||
config.kv_transfer_config = None
|
||||
if kv_connector_extra_config is not None:
|
||||
config.kv_transfer_config = MagicMock()
|
||||
config.kv_transfer_config.kv_connector_extra_config = kv_connector_extra_config
|
||||
return config
|
||||
|
||||
|
||||
def _run(n_c4=3, n_c128=2, n_swa=5, mem=100 * 1024 * 1024):
|
||||
groups = _make_groups(n_c4, n_c128, n_swa)
|
||||
return _get_kv_cache_config_deepseek_v4(_mock_vllm_config(), groups, mem)
|
||||
return _get_kv_cache_config_packed(_mock_vllm_config(), groups, mem)
|
||||
|
||||
|
||||
def _page_sizes_by_layer(
|
||||
@@ -135,7 +138,7 @@ class TestInterleavedPacking:
|
||||
def test_strided_views_are_independent(self):
|
||||
groups = _make_groups(n_c4=3, n_c128=2, n_swa=5)
|
||||
page_sizes = _page_sizes_by_layer(groups)
|
||||
num_blocks, tensors = _get_kv_cache_config_deepseek_v4(
|
||||
num_blocks, tensors = _get_kv_cache_config_packed(
|
||||
_mock_vllm_config(), groups, 100 * 1024 * 1024
|
||||
)
|
||||
backing = torch.zeros(tensors[0].size, dtype=torch.uint8)
|
||||
@@ -156,8 +159,7 @@ class TestInterleavedPacking:
|
||||
for i, v in enumerate(views):
|
||||
assert (v == i + 1).all(), f"View {i} was corrupted"
|
||||
|
||||
def test_hma_attention_groups_keep_default_backing(self, monkeypatch):
|
||||
monkeypatch.setattr(envs, "VLLM_USE_PACKED_HMA_KV_CACHE", False, raising=False)
|
||||
def test_hma_attention_groups_keep_default_backing(self):
|
||||
full = _make_full_spec()
|
||||
sw = _make_sw_spec()
|
||||
page_size = full.page_size_bytes
|
||||
@@ -178,8 +180,7 @@ class TestInterleavedPacking:
|
||||
KVCacheTensor(size=page_size * 32, shared_by=["full.1", "sw.2", "sw.3"]),
|
||||
]
|
||||
|
||||
def test_hma_attention_groups_use_packed_backing_with_flag(self, monkeypatch):
|
||||
monkeypatch.setattr(envs, "VLLM_USE_PACKED_HMA_KV_CACHE", True, raising=False)
|
||||
def test_hma_attention_groups_use_packed_backing_with_enable_cross_layers(self):
|
||||
full = _make_full_spec()
|
||||
sw = _make_sw_spec()
|
||||
page_size = full.page_size_bytes
|
||||
@@ -190,7 +191,9 @@ class TestInterleavedPacking:
|
||||
]
|
||||
|
||||
config = get_kv_cache_config_from_groups(
|
||||
_mock_vllm_config(), groups, available_memory=page_size * 2 * 32
|
||||
_mock_vllm_config({"enable_cross_layers_blocks": "True"}),
|
||||
groups,
|
||||
available_memory=page_size * 2 * 32,
|
||||
)
|
||||
|
||||
assert config.num_blocks == 32
|
||||
|
||||
@@ -124,13 +124,14 @@ class OffloadingConnectorWorker:
|
||||
raise NotImplementedError
|
||||
|
||||
packed_kv_cache_tensor = next(
|
||||
(t for t in kv_cache_config.kv_cache_tensors if t.block_stride), None
|
||||
(
|
||||
t
|
||||
for t in kv_cache_config.kv_cache_tensors
|
||||
if t.block_stride and t.shared_by
|
||||
),
|
||||
None,
|
||||
)
|
||||
is_dsv4 = all(
|
||||
isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
|
||||
for group in kv_cache_config.kv_cache_groups
|
||||
)
|
||||
if packed_kv_cache_tensor is not None and not is_dsv4:
|
||||
if packed_kv_cache_tensor is not None:
|
||||
(tensor,) = tensors_per_block[packed_kv_cache_tensor.shared_by[0]]
|
||||
block_stride = tensor.stride(0)
|
||||
packed_tensor = tensor.as_strided(
|
||||
@@ -153,7 +154,7 @@ class OffloadingConnectorWorker:
|
||||
block_data_refs: dict[str, list[CanonicalKVCacheRef]] = defaultdict(list)
|
||||
for kv_cache_tensor in kv_cache_config.kv_cache_tensors:
|
||||
# Filter to layers that were actually processed above.
|
||||
# _get_kv_cache_config_deepseek_v4 emits KVCacheTensor entries for
|
||||
# Packed KV allocation emits KVCacheTensor entries for
|
||||
# every (tuple_idx, page_size) slot; slots where no group has a
|
||||
# layer at that index produce an empty shared_by (reserved memory
|
||||
# with no corresponding model layer).
|
||||
|
||||
@@ -210,7 +210,6 @@ if TYPE_CHECKING:
|
||||
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: int = 300
|
||||
VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS: int = 5
|
||||
VLLM_KV_CACHE_LAYOUT: Literal["NHD", "HND"] | None = None
|
||||
VLLM_USE_PACKED_HMA_KV_CACHE: bool = False
|
||||
VLLM_SSM_CONV_STATE_LAYOUT: Literal["SD", "DS"] | None = None
|
||||
VLLM_COMPUTE_NANS_IN_LOGITS: bool = False
|
||||
VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: Literal[
|
||||
@@ -1614,11 +1613,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_KV_CACHE_LAYOUT": env_with_choices(
|
||||
"VLLM_KV_CACHE_LAYOUT", None, ["NHD", "HND"]
|
||||
),
|
||||
# Opt into packed per-block KV cache allocation for multi-group
|
||||
# attention-only HMA models (e.g. gpt-oss, Gemma 3/4).
|
||||
"VLLM_USE_PACKED_HMA_KV_CACHE": lambda: bool(
|
||||
int(os.getenv("VLLM_USE_PACKED_HMA_KV_CACHE", "0"))
|
||||
),
|
||||
# SSM conv state layout used for Mamba models.
|
||||
# - SD: (state_len, dim) — dim contiguous (default)
|
||||
# - DS: (dim, state_len) — TP-sharded dim on dim1,
|
||||
|
||||
@@ -947,7 +947,9 @@ def may_override_num_blocks(vllm_config: VllmConfig, num_blocks: int) -> int:
|
||||
return num_blocks
|
||||
|
||||
|
||||
def _pool_bytes_per_block(kv_cache_groups: list[KVCacheGroupSpec]) -> int:
|
||||
def _pool_bytes_per_block(
|
||||
vllm_config: VllmConfig, kv_cache_groups: list[KVCacheGroupSpec]
|
||||
) -> int:
|
||||
"""
|
||||
Bytes consumed by one block in the worker's shared KV cache pool, mirroring
|
||||
the divisor used by `get_kv_cache_config_from_groups` to convert
|
||||
@@ -958,7 +960,7 @@ def _pool_bytes_per_block(kv_cache_groups: list[KVCacheGroupSpec]) -> int:
|
||||
kv_cache_groups[0].kv_cache_spec, UniformTypeKVCacheSpecs
|
||||
):
|
||||
return kv_cache_groups[0].kv_cache_spec.page_size_bytes
|
||||
if _use_packed_kv_cache_groups(kv_cache_groups):
|
||||
if _use_packed_kv_cache_config(vllm_config, kv_cache_groups):
|
||||
# buckets = {page_size: [[layer_names], [layer_names], ...]}
|
||||
buckets = _bucket_layers_by_page_size(kv_cache_groups)
|
||||
return sum(ps * len(slots) for ps, slots in buckets.items())
|
||||
@@ -1250,16 +1252,26 @@ def _bucket_layers_by_page_size(
|
||||
return buckets
|
||||
|
||||
|
||||
def _use_packed_kv_cache_groups(
|
||||
def _use_packed_kv_cache_config(
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
) -> bool:
|
||||
is_dsv4 = all(
|
||||
isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
|
||||
for group in kv_cache_groups
|
||||
)
|
||||
return is_dsv4 or (
|
||||
bool(envs.VLLM_USE_PACKED_HMA_KV_CACHE) and len(kv_cache_groups) > 1
|
||||
kv_transfer_config = vllm_config.kv_transfer_config
|
||||
extra_config = (
|
||||
kv_transfer_config.kv_connector_extra_config
|
||||
if kv_transfer_config is not None
|
||||
else {}
|
||||
)
|
||||
# NOTE: enable_cross_layers_blocks is an experimental API and subject to change with
|
||||
# https://github.com/vllm-project/vllm/issues/42082
|
||||
enable_cross_layers = (
|
||||
str(extra_config.get("enable_cross_layers_blocks", "False")).lower() == "true"
|
||||
)
|
||||
return is_dsv4 or (enable_cross_layers and len(kv_cache_groups) > 1)
|
||||
|
||||
|
||||
def _get_kv_cache_config_packed(
|
||||
@@ -1347,10 +1359,9 @@ def get_kv_cache_config_from_groups(
|
||||
)
|
||||
for layer_name in kv_cache_groups[0].layer_names
|
||||
]
|
||||
elif _use_packed_kv_cache_groups(kv_cache_groups):
|
||||
# DeepSeek V4 keeps the existing packed layout. Other multi-group
|
||||
# attention-only HMA layouts can opt in with
|
||||
# VLLM_USE_PACKED_HMA_KV_CACHE=1.
|
||||
elif _use_packed_kv_cache_config(vllm_config, kv_cache_groups):
|
||||
# DeepSeek V4 uses the packed layout by default. Other multi-group
|
||||
# layouts can opt in with --enable-cross-layers.
|
||||
num_blocks, kv_cache_tensors = _get_kv_cache_config_packed(
|
||||
vllm_config, kv_cache_groups, available_memory
|
||||
)
|
||||
@@ -2069,7 +2080,7 @@ def get_kv_cache_configs(
|
||||
if not groups:
|
||||
adjusted_memory.append(avail_mem)
|
||||
continue
|
||||
bytes_per_block = _pool_bytes_per_block(groups)
|
||||
bytes_per_block = _pool_bytes_per_block(vllm_config, groups)
|
||||
logger.info(
|
||||
"Overriding num_gpu_blocks=%d with num_gpu_blocks_override=%d",
|
||||
avail_mem // bytes_per_block,
|
||||
|
||||
Reference in New Issue
Block a user