diff --git a/tests/v1/core/test_contiguous_kv_packing.py b/tests/v1/core/test_contiguous_kv_packing.py index f4b7ee520a..647241ce73 100644 --- a/tests/v1/core/test_contiguous_kv_packing.py +++ b/tests/v1/core/test_contiguous_kv_packing.py @@ -7,9 +7,8 @@ from unittest.mock import MagicMock import pytest import torch -from vllm import envs from vllm.v1.core.kv_cache_utils import ( - _get_kv_cache_config_deepseek_v4, + _get_kv_cache_config_packed, get_kv_cache_config_from_groups, ) from vllm.v1.kv_cache_interface import ( @@ -84,15 +83,19 @@ def _make_groups(n_c4, n_c128, n_swa): return [mla_group, swa_group] -def _mock_vllm_config(): +def _mock_vllm_config(kv_connector_extra_config: dict[str, str] | None = None): config = MagicMock() config.cache_config.num_gpu_blocks_override = None + config.kv_transfer_config = None + if kv_connector_extra_config is not None: + config.kv_transfer_config = MagicMock() + config.kv_transfer_config.kv_connector_extra_config = kv_connector_extra_config return config def _run(n_c4=3, n_c128=2, n_swa=5, mem=100 * 1024 * 1024): groups = _make_groups(n_c4, n_c128, n_swa) - return _get_kv_cache_config_deepseek_v4(_mock_vllm_config(), groups, mem) + return _get_kv_cache_config_packed(_mock_vllm_config(), groups, mem) def _page_sizes_by_layer( @@ -135,7 +138,7 @@ class TestInterleavedPacking: def test_strided_views_are_independent(self): groups = _make_groups(n_c4=3, n_c128=2, n_swa=5) page_sizes = _page_sizes_by_layer(groups) - num_blocks, tensors = _get_kv_cache_config_deepseek_v4( + num_blocks, tensors = _get_kv_cache_config_packed( _mock_vllm_config(), groups, 100 * 1024 * 1024 ) backing = torch.zeros(tensors[0].size, dtype=torch.uint8) @@ -156,8 +159,7 @@ class TestInterleavedPacking: for i, v in enumerate(views): assert (v == i + 1).all(), f"View {i} was corrupted" - def test_hma_attention_groups_keep_default_backing(self, monkeypatch): - monkeypatch.setattr(envs, "VLLM_USE_PACKED_HMA_KV_CACHE", False, raising=False) + def test_hma_attention_groups_keep_default_backing(self): full = _make_full_spec() sw = _make_sw_spec() page_size = full.page_size_bytes @@ -178,8 +180,7 @@ class TestInterleavedPacking: KVCacheTensor(size=page_size * 32, shared_by=["full.1", "sw.2", "sw.3"]), ] - def test_hma_attention_groups_use_packed_backing_with_flag(self, monkeypatch): - monkeypatch.setattr(envs, "VLLM_USE_PACKED_HMA_KV_CACHE", True, raising=False) + def test_hma_attention_groups_use_packed_backing_with_enable_cross_layers(self): full = _make_full_spec() sw = _make_sw_spec() page_size = full.page_size_bytes @@ -190,7 +191,9 @@ class TestInterleavedPacking: ] config = get_kv_cache_config_from_groups( - _mock_vllm_config(), groups, available_memory=page_size * 2 * 32 + _mock_vllm_config({"enable_cross_layers_blocks": "True"}), + groups, + available_memory=page_size * 2 * 32, ) assert config.num_blocks == 32 diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py index 1e0435d371..254e0dec09 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py @@ -124,13 +124,14 @@ class OffloadingConnectorWorker: raise NotImplementedError packed_kv_cache_tensor = next( - (t for t in kv_cache_config.kv_cache_tensors if t.block_stride), None + ( + t + for t in kv_cache_config.kv_cache_tensors + if t.block_stride and t.shared_by + ), + None, ) - is_dsv4 = all( - isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs) - for group in kv_cache_config.kv_cache_groups - ) - if packed_kv_cache_tensor is not None and not is_dsv4: + if packed_kv_cache_tensor is not None: (tensor,) = tensors_per_block[packed_kv_cache_tensor.shared_by[0]] block_stride = tensor.stride(0) packed_tensor = tensor.as_strided( @@ -153,7 +154,7 @@ class OffloadingConnectorWorker: block_data_refs: dict[str, list[CanonicalKVCacheRef]] = defaultdict(list) for kv_cache_tensor in kv_cache_config.kv_cache_tensors: # Filter to layers that were actually processed above. - # _get_kv_cache_config_deepseek_v4 emits KVCacheTensor entries for + # Packed KV allocation emits KVCacheTensor entries for # every (tuple_idx, page_size) slot; slots where no group has a # layer at that index produce an empty shared_by (reserved memory # with no corresponding model layer). diff --git a/vllm/envs.py b/vllm/envs.py index 9cfd4792e1..bd82a4069c 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -210,7 +210,6 @@ if TYPE_CHECKING: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: int = 300 VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS: int = 5 VLLM_KV_CACHE_LAYOUT: Literal["NHD", "HND"] | None = None - VLLM_USE_PACKED_HMA_KV_CACHE: bool = False VLLM_SSM_CONV_STATE_LAYOUT: Literal["SD", "DS"] | None = None VLLM_COMPUTE_NANS_IN_LOGITS: bool = False VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: Literal[ @@ -1614,11 +1613,6 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_KV_CACHE_LAYOUT": env_with_choices( "VLLM_KV_CACHE_LAYOUT", None, ["NHD", "HND"] ), - # Opt into packed per-block KV cache allocation for multi-group - # attention-only HMA models (e.g. gpt-oss, Gemma 3/4). - "VLLM_USE_PACKED_HMA_KV_CACHE": lambda: bool( - int(os.getenv("VLLM_USE_PACKED_HMA_KV_CACHE", "0")) - ), # SSM conv state layout used for Mamba models. # - SD: (state_len, dim) — dim contiguous (default) # - DS: (dim, state_len) — TP-sharded dim on dim1, diff --git a/vllm/v1/core/kv_cache_utils.py b/vllm/v1/core/kv_cache_utils.py index a3822e7fc4..b13c23d804 100644 --- a/vllm/v1/core/kv_cache_utils.py +++ b/vllm/v1/core/kv_cache_utils.py @@ -947,7 +947,9 @@ def may_override_num_blocks(vllm_config: VllmConfig, num_blocks: int) -> int: return num_blocks -def _pool_bytes_per_block(kv_cache_groups: list[KVCacheGroupSpec]) -> int: +def _pool_bytes_per_block( + vllm_config: VllmConfig, kv_cache_groups: list[KVCacheGroupSpec] +) -> int: """ Bytes consumed by one block in the worker's shared KV cache pool, mirroring the divisor used by `get_kv_cache_config_from_groups` to convert @@ -958,7 +960,7 @@ def _pool_bytes_per_block(kv_cache_groups: list[KVCacheGroupSpec]) -> int: kv_cache_groups[0].kv_cache_spec, UniformTypeKVCacheSpecs ): return kv_cache_groups[0].kv_cache_spec.page_size_bytes - if _use_packed_kv_cache_groups(kv_cache_groups): + if _use_packed_kv_cache_config(vllm_config, kv_cache_groups): # buckets = {page_size: [[layer_names], [layer_names], ...]} buckets = _bucket_layers_by_page_size(kv_cache_groups) return sum(ps * len(slots) for ps, slots in buckets.items()) @@ -1250,16 +1252,26 @@ def _bucket_layers_by_page_size( return buckets -def _use_packed_kv_cache_groups( +def _use_packed_kv_cache_config( + vllm_config: VllmConfig, kv_cache_groups: list[KVCacheGroupSpec], ) -> bool: is_dsv4 = all( isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs) for group in kv_cache_groups ) - return is_dsv4 or ( - bool(envs.VLLM_USE_PACKED_HMA_KV_CACHE) and len(kv_cache_groups) > 1 + kv_transfer_config = vllm_config.kv_transfer_config + extra_config = ( + kv_transfer_config.kv_connector_extra_config + if kv_transfer_config is not None + else {} ) + # NOTE: enable_cross_layers_blocks is an experimental API and subject to change with + # https://github.com/vllm-project/vllm/issues/42082 + enable_cross_layers = ( + str(extra_config.get("enable_cross_layers_blocks", "False")).lower() == "true" + ) + return is_dsv4 or (enable_cross_layers and len(kv_cache_groups) > 1) def _get_kv_cache_config_packed( @@ -1347,10 +1359,9 @@ def get_kv_cache_config_from_groups( ) for layer_name in kv_cache_groups[0].layer_names ] - elif _use_packed_kv_cache_groups(kv_cache_groups): - # DeepSeek V4 keeps the existing packed layout. Other multi-group - # attention-only HMA layouts can opt in with - # VLLM_USE_PACKED_HMA_KV_CACHE=1. + elif _use_packed_kv_cache_config(vllm_config, kv_cache_groups): + # DeepSeek V4 uses the packed layout by default. Other multi-group + # layouts can opt in with --enable-cross-layers. num_blocks, kv_cache_tensors = _get_kv_cache_config_packed( vllm_config, kv_cache_groups, available_memory ) @@ -2069,7 +2080,7 @@ def get_kv_cache_configs( if not groups: adjusted_memory.append(avail_mem) continue - bytes_per_block = _pool_bytes_per_block(groups) + bytes_per_block = _pool_bytes_per_block(vllm_config, groups) logger.info( "Overriding num_gpu_blocks=%d with num_gpu_blocks_override=%d", avail_mem // bytes_per_block,