From 017e9f4448b700e85ee16023287b025693c72b9e Mon Sep 17 00:00:00 2001 From: Tyler Michael Smith Date: Mon, 17 Aug 2026 09:20:50 -0400 Subject: [PATCH] Promote `prefix_cache_retention_interval` to an argument and change the default to 0 (#52216) Signed-off-by: Tyler Michael Smith Co-authored-by: OpenAI Codex --- tests/config/test_config_utils.py | 5 + tests/engine/test_arg_utils.py | 23 ++++ tests/v1/core/test_contiguous_kv_packing.py | 2 + tests/v1/core/test_kv_cache_utils.py | 4 + tests/v1/core/test_prefix_caching.py | 110 +++++++++++------- tests/v1/engine/test_engine_args.py | 5 + tests/v1/engine/test_engine_core_client.py | 1 + .../unit/test_mooncake_store_worker.py | 6 +- vllm/config/cache.py | 23 +++- .../kv_connector/v1/mooncake/store/worker.py | 2 +- vllm/engine/arg_utils.py | 8 ++ vllm/envs.py | 6 - vllm/v1/core/kv_cache_coordinator.py | 11 +- vllm/v1/core/kv_cache_manager.py | 2 +- vllm/v1/core/kv_cache_utils.py | 6 + vllm/v1/kv_cache_interface.py | 2 + vllm/v1/simple_kv_offload/manager.py | 7 +- 17 files changed, 160 insertions(+), 63 deletions(-) diff --git a/tests/config/test_config_utils.py b/tests/config/test_config_utils.py index 35bc1e167b..24ef1b52a9 100644 --- a/tests/config/test_config_utils.py +++ b/tests/config/test_config_utils.py @@ -216,6 +216,11 @@ def test_cache_config_hash_ignores_kv_cache_sizing_knobs(): assert CacheConfig(gpu_memory_utilization=0.5).compute_hash() == base_hash +def test_cache_config_hash_ignores_prefix_cache_retention_interval(): + base_hash = CacheConfig().compute_hash() + assert CacheConfig(prefix_cache_retention_interval=64).compute_hash() == base_hash + + def test_envs_compile_factors_relocation_invariant(tmp_path): """Relocating HOME or the XDG roots must not change the compile-cache env hash. diff --git a/tests/engine/test_arg_utils.py b/tests/engine/test_arg_utils.py index 2feb9f7a03..6a799ee912 100644 --- a/tests/engine/test_arg_utils.py +++ b/tests/engine/test_arg_utils.py @@ -476,6 +476,7 @@ def test_prefix_cache_default(): # should be None by default (depends on model). engine_args = EngineArgs.from_cli_args(args=args) assert engine_args.enable_prefix_caching is None + assert engine_args.prefix_cache_retention_interval == 0 # with flag to turn it on. args = parser.parse_args(["--enable-prefix-caching"]) @@ -487,6 +488,28 @@ def test_prefix_cache_default(): engine_args = EngineArgs.from_cli_args(args=args) assert not engine_args.enable_prefix_caching + args = parser.parse_args(["--prefix-cache-retention-interval", "64"]) + engine_args = EngineArgs.from_cli_args(args=args) + assert engine_args.prefix_cache_retention_interval == 64 + + +def test_prefix_cache_retention_interval_from_deprecated_env( + monkeypatch, caplog, disable_log_dedup +): + monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "64") + + engine_args = EngineArgs() + + assert engine_args.prefix_cache_retention_interval == 64 + assert "VLLM_PREFIX_CACHE_RETENTION_INTERVAL" in caplog.text + assert "deprecated" in caplog.text + assert "prefix_cache_retention_interval" in caplog.text + + parser = EngineArgs.add_cli_args(FlexibleArgumentParser()) + args = parser.parse_args(["--prefix-cache-retention-interval", "32"]) + engine_args = EngineArgs.from_cli_args(args) + assert engine_args.prefix_cache_retention_interval == 32 + @pytest.mark.parametrize( ("arg", "expected", "option"), diff --git a/tests/v1/core/test_contiguous_kv_packing.py b/tests/v1/core/test_contiguous_kv_packing.py index 88d17e9acd..40906f6213 100644 --- a/tests/v1/core/test_contiguous_kv_packing.py +++ b/tests/v1/core/test_contiguous_kv_packing.py @@ -88,6 +88,7 @@ def _make_groups(n_c4, n_c128, n_swa): def _mock_vllm_config(kv_connector_extra_config: dict[str, str] | None = None): config = MagicMock() config.cache_config.num_gpu_blocks_override = None + config.cache_config.prefix_cache_retention_interval = 0 config.kv_transfer_config = None if kv_connector_extra_config is not None: config.kv_transfer_config = MagicMock() @@ -322,6 +323,7 @@ class TestInterleavedPacking: ) assert config.num_blocks == 32 + assert config.prefix_cache_retention_interval == 0 assert sum(t.size for t in config.kv_cache_tensors) == page_size * 2 * 32 assert config.kv_cache_tensors == [ KVCacheTensor(size=page_size * 32, shared_by=["full.0", "sw.0", "sw.1"]), diff --git a/tests/v1/core/test_kv_cache_utils.py b/tests/v1/core/test_kv_cache_utils.py index 9b2afeb001..bfe4996b1a 100644 --- a/tests/v1/core/test_kv_cache_utils.py +++ b/tests/v1/core/test_kv_cache_utils.py @@ -816,6 +816,7 @@ def test_metrics_empty_stats(): def test_get_kv_cache_configs_multiple_workers(): model_config = ModelConfig(max_model_len=16) vllm_config = VllmConfig(model_config=model_config) + vllm_config.cache_config.prefix_cache_retention_interval = None ref_kv_cache_spec = new_kv_cache_spec() same_kv_cache_specs = [ @@ -1173,6 +1174,7 @@ def test_get_kv_cache_configs_multiple_workers(): def test_get_kv_cache_configs_pp_sharding(asymmetric_memory): model_config = ModelConfig(max_model_len=512) vllm_config = VllmConfig(model_config=model_config) + vllm_config.cache_config.prefix_cache_retention_interval = None ref_kv_cache_spec = new_kv_cache_spec() pp_kv_cache_specs = [ @@ -1702,6 +1704,7 @@ def test_get_kv_cache_config_one_worker(): # pass max_model_len to pass check_enough_kv_cache_memory model_config = ModelConfig(max_model_len=16) vllm_config = VllmConfig(model_config=model_config) + vllm_config.cache_config.prefix_cache_retention_interval = None mem_per_block_per_layer = 16 * 2 * 64 * 4 * 2 # all layers are full attention -> single group @@ -2015,6 +2018,7 @@ def test_get_kv_cache_config_one_worker(): def test_get_kv_cache_configs_attention_free(): kv_cache_specs: dict[str, KVCacheSpec] = {} vllm_config = VllmConfig(model_config=ModelConfig(max_model_len=16)) + vllm_config.cache_config.prefix_cache_retention_interval = None kv_cache_configs = get_kv_cache_configs(vllm_config, [kv_cache_specs], [0]) assert kv_cache_configs == [ KVCacheConfig( diff --git a/tests/v1/core/test_prefix_caching.py b/tests/v1/core/test_prefix_caching.py index 13ed7c7b9d..6cd117b1fa 100644 --- a/tests/v1/core/test_prefix_caching.py +++ b/tests/v1/core/test_prefix_caching.py @@ -4,6 +4,7 @@ import copy from collections.abc import Callable +from dataclasses import replace from math import lcm from types import SimpleNamespace @@ -110,6 +111,11 @@ def make_kv_cache_manager(kv_cache_config: KVCacheConfig, **kwargs) -> KVCacheMa "scheduler_block_size", lcm(*(g.kv_cache_spec.block_size for g in kv_cache_config.kv_cache_groups)), ) + if "retention_interval" in kwargs: + kv_cache_config = replace( + kv_cache_config, + prefix_cache_retention_interval=kwargs.pop("retention_interval"), + ) return KVCacheManager(kv_cache_config, **kwargs) @@ -3119,9 +3125,8 @@ def test_hybrid_cache_blocks_clamped_to_lcm(): ) -def test_hybrid_local_kv_retention_interval_aligns_in_manager(monkeypatch): +def test_hybrid_local_kv_retention_interval_aligns_in_manager(): """Verify fixed intervals retain sparse tails plus the latest replay tail.""" - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "64") block_size = 8 kv_cache_config = KVCacheConfig( num_blocks=100, @@ -3153,6 +3158,7 @@ def test_hybrid_local_kv_retention_interval_aligns_in_manager(monkeypatch): max_model_len=8192, enable_caching=True, hash_block_size=block_size, + retention_interval=64, ) # The SWA manager uses the configured 64-token interval (a multiple of the @@ -3184,18 +3190,15 @@ def test_hybrid_local_kv_retention_interval_aligns_in_manager(monkeypatch): "interval, expected_match", [ # scheduler_block_size is 32 (= lcm(4*8, 8)); 33 is not a multiple of it. - ("33", "multiple of scheduler_block_size"), + (33, "multiple of scheduler_block_size"), # A negative multiple (-32 % 32 == 0) must still be rejected explicitly, # otherwise it would pass the modulo check and silently degrade to dense. - ("-32", "non-negative"), + (-32, "non-negative"), ], ) -def test_hybrid_local_kv_retention_interval_rejects_invalid( - monkeypatch, interval, expected_match -): +def test_hybrid_local_kv_retention_interval_rejects_invalid(interval, expected_match): """A retention interval that is negative or not a multiple of scheduler_block_size errors out at construction time.""" - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", interval) block_size = 8 kv_cache_config = KVCacheConfig( num_blocks=100, @@ -3228,12 +3231,36 @@ def test_hybrid_local_kv_retention_interval_rejects_invalid( max_model_len=8192, enable_caching=True, hash_block_size=block_size, + retention_interval=interval, ) -def test_hybrid_local_kv_retention_interval_survives_recycling(monkeypatch): +def test_zero_retention_is_ignored_for_full_attention(): + kv_cache_config = make_kv_cache_config(block_size=16, num_blocks=10) + manager = make_kv_cache_manager( + kv_cache_config, + max_model_len=8192, + enable_caching=True, + hash_block_size=16, + retention_interval=0, + ) + assert manager.coordinator.retention_interval == 0 + + +def test_positive_retention_rejects_full_attention(): + kv_cache_config = make_kv_cache_config(block_size=16, num_blocks=10) + with pytest.raises(ValueError, match="no sliding-window or Mamba"): + make_kv_cache_manager( + kv_cache_config, + max_model_len=8192, + enable_caching=True, + hash_block_size=16, + retention_interval=16, + ) + + +def test_hybrid_local_kv_retention_interval_survives_recycling(): """Verify retained local checkpoints are reused after block recycling.""" - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "1024") hash_block_size = 4 kv_cache_config = KVCacheConfig( num_blocks=800, @@ -3286,6 +3313,7 @@ def test_hybrid_local_kv_retention_interval_survives_recycling(monkeypatch): max_model_len=4096, enable_caching=True, hash_block_size=hash_block_size, + retention_interval=1024, ) def fill_request(request_id: str, token_offset: int) -> list[int]: @@ -3314,9 +3342,8 @@ def test_hybrid_local_kv_retention_interval_survives_recycling(monkeypatch): assert [len(blocks) for blocks in computed_blocks.blocks] == [4, 16, 128, 256] -def test_hybrid_local_kv_retention_latest_only_reuses_replay_boundary(monkeypatch): +def test_hybrid_local_kv_retention_latest_only_reuses_replay_boundary(): """Verify latest-only retention reuses only the replayable prompt boundary.""" - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "0") block_size = 8 kv_cache_config = KVCacheConfig( num_blocks=100, @@ -3348,6 +3375,7 @@ def test_hybrid_local_kv_retention_latest_only_reuses_replay_boundary(monkeypatc max_model_len=8192, enable_caching=True, hash_block_size=block_size, + retention_interval=0, ) token_ids = [i for i in range(16) for _ in range(block_size)] @@ -3388,14 +3416,13 @@ def test_hybrid_local_kv_retention_latest_only_reuses_replay_boundary(monkeypatc assert len(computed_blocks.blocks[1]) == 0 -def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary(monkeypatch): +def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary(): """Verify MTP/EAGLE SWA retention keeps the extra proof block. EAGLE/MTP lookup matches one additional local block after the returned prefix and then drops it. Sparse retention must therefore cache the normal local tail at the latest replay boundary plus one extra SWA block. """ - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "0") block_size = 8 kv_cache_config = KVCacheConfig( num_blocks=100, @@ -3428,6 +3455,7 @@ def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary(monkeypatch): max_model_len=8192, enable_caching=True, hash_block_size=block_size, + retention_interval=0, use_eagle=True, ) @@ -3824,12 +3852,11 @@ def test_cache_hit_local_and_external_two_groups_preempt_and_reallocate(): assert manager.get_blocks("test").get_block_ids() != ([], []) -def test_swa_free_split_keeps_cached_tail_ahead_of_scratch(monkeypatch): - """Default path (no retention): freeing an SWA request must place its +def test_swa_free_split_keeps_cached_tail_ahead_of_scratch(): + """Dense retention: freeing an SWA request must place its uncached scratch blocks at the front of the free queue (recycled first) and keep its cached checkpoint blocks at the back (retained for prefix hits). This split is always-on, independent of the retention interval.""" - monkeypatch.delenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", raising=False) block_size = 8 kv_cache_config = KVCacheConfig( num_blocks=100, @@ -3937,13 +3964,14 @@ def _make_pure_swa_manager(block_size, sliding_window, num_blocks=100, **kwargs) ) -def test_pure_swa_retention_interval_caches_sparse_tails(monkeypatch): +def test_pure_swa_retention_interval_caches_sparse_tails(): """Sparse retention must work for a pure-SWA single-group model, not just hybrid models: only the per-interval tails plus the latest replay tail are cached, and a replay still hits the latest replayable boundary.""" - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "64") block_size = 16 - manager = _make_pure_swa_manager(block_size, sliding_window=block_size) + manager = _make_pure_swa_manager( + block_size, sliding_window=block_size, retention_interval=64 + ) assert type(manager.coordinator).__name__ == "UnitaryKVCacheCoordinator" token_ids = [i for i in range(16) for _ in range(block_size)] @@ -3976,11 +4004,12 @@ def test_pure_swa_retention_interval_caches_sparse_tails(monkeypatch): assert num_computed == 240 -def test_pure_swa_retention_latest_only(monkeypatch): +def test_pure_swa_retention_latest_only(): """`=0` on a pure-SWA model keeps only the latest replay tail.""" - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "0") block_size = 16 - manager = _make_pure_swa_manager(block_size, sliding_window=block_size) + manager = _make_pure_swa_manager( + block_size, sliding_window=block_size, retention_interval=0 + ) token_ids = [i for i in range(16) for _ in range(block_size)] req = make_request("0", token_ids, block_size, sha256) @@ -4008,10 +4037,9 @@ def test_pure_swa_retention_latest_only(monkeypatch): assert num_computed == 240 -def test_pure_swa_retention_dense_default_caches_all(monkeypatch): - """With retention unset, a pure-SWA model must keep the dense behavior: +def test_pure_swa_dense_retention_caches_all(): + """With retention set to ``None``, a pure-SWA model keeps dense behavior: every block boundary is a potential hit, so all blocks are cached.""" - monkeypatch.delenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", raising=False) block_size = 16 manager = _make_pure_swa_manager(block_size, sliding_window=block_size) @@ -4037,7 +4065,7 @@ def test_pure_swa_retention_dense_default_caches_all(monkeypatch): def test_mamba_reachable_block_mask_sparsifies_retention(): - """Mamba state-snapshot retention: with VLLM_PREFIX_CACHE_RETENTION_INTERVAL + """Mamba state-snapshot retention: with a configured retention interval, the manager keeps one cached state per interval-sized segment (plus the latest replay boundary) instead of a snapshot per block, which is what lets a small attention block_size avoid Mamba dominating the KV pool.""" @@ -4063,7 +4091,7 @@ def test_mamba_reachable_block_mask_sparsifies_retention(): ) return None if m is None else {i for i, v in enumerate(m) if v} - # Dense default (None) -> no mask, every block cached (unchanged behavior). + # Dense retention (None) -> no mask, every block cached. assert retained(None) is None # interval == block_size -> every block is a boundary -> stays dense. assert retained(block_size) is None @@ -4111,7 +4139,7 @@ def test_mamba_reachable_block_mask_pins_shared_prefix(): assert retained(0, 100) == {5, 14} # Coexists with segment tails (interval 64 -> {3,7,11,15} + replay 14). assert retained(64, 96) == {3, 5, 7, 11, 14, 15} - # Dense default ignores the hint (nothing to sparsify). + # Dense retention ignores the hint (nothing to sparsify). assert retained(None, 96) is None # Out-of-range boundary is a no-op (only replay 14 remains). assert retained(0, 16 * block_size * 2) == {14} @@ -4120,14 +4148,13 @@ def test_mamba_reachable_block_mask_pins_shared_prefix(): assert retained(0, None) == {14} -def test_mamba_shared_prefix_survives_zero_retention(monkeypatch): +def test_mamba_shared_prefix_survives_zero_retention(): """Manager-level check of the full wiring: a pinned shared-prefix boundary (``Request.shared_prefix_boundary``, set by the scheduler on Marconi-style detection) keeps its Mamba state block cached under - ``VLLM_PREFIX_CACHE_RETENTION_INTERVAL=0``, which otherwise retains only the + ``prefix_cache_retention_interval=0``, which otherwise retains only the end-of-prompt replay boundary. Without this, a shared prefix (junction before ``num_prompt``) would be recomputed by every sharing request.""" - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "0") block_size = 16 # 16-block (256-token) prompt; replay boundary is block 240 // 16 - 1 = 14. @@ -4140,6 +4167,7 @@ def test_mamba_shared_prefix_survives_zero_retention(monkeypatch): max_model_len=8192, enable_caching=True, hash_block_size=block_size, + retention_interval=0, ) req = make_request("r", token_ids, block_size, sha256) req.shared_prefix_boundary = shared_prefix_boundary @@ -4163,24 +4191,21 @@ def test_mamba_shared_prefix_survives_zero_retention(monkeypatch): assert cached_mamba_blocks(96) == {5, 14} -def test_mamba_shared_prefix_reuse_under_zero_retention(monkeypatch): +def test_mamba_shared_prefix_reuse_under_zero_retention(): """Full cross-request Marconi flow: a partial shared prefix cached by the detecting request must stay reusable by a later request under - ``VLLM_PREFIX_CACHE_RETENTION_INTERVAL=0``. Without the pin the junction is + ``prefix_cache_retention_interval=0``. Without the pin the junction is masked out and the later request misses; with it (and under dense) the reuse is preserved.""" block_size = 16 def last_req_hit(retention, pin): - if retention is None: - monkeypatch.delenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", raising=False) - else: - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", str(retention)) manager = make_kv_cache_manager( _make_hybrid_kv_cache_config(block_size, 200, ["full", "mamba_align"]), max_model_len=8192, enable_caching=True, hash_block_size=block_size, + retention_interval=retention, ) shared = [7 for _ in range(2 * block_size)] # 2-block shared prefix @@ -4261,23 +4286,20 @@ def test_swa_reachable_block_mask_pins_shared_prefix(): assert retained(0, 0, block_size) == {14} -def test_swa_shared_prefix_reuse_under_zero_retention(monkeypatch): +def test_swa_shared_prefix_reuse_under_zero_retention(): """SWA cross-request analog: a partial shared prefix's sliding-window tail - must stay reusable under ``VLLM_PREFIX_CACHE_RETENTION_INTERVAL=0``. Without + must stay reusable under ``prefix_cache_retention_interval=0``. Without the pin the junction window is masked out and a later request misses; with it (and under dense) reuse is preserved.""" block_size = 16 def last_req_hit(retention, pin): - if retention is None: - monkeypatch.delenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", raising=False) - else: - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", str(retention)) manager = make_kv_cache_manager( _make_hybrid_kv_cache_config(block_size, 200, ["full", "sliding_window"]), max_model_len=8192, enable_caching=True, hash_block_size=block_size, + retention_interval=retention, ) shared = [7 for _ in range(4 * block_size)] # 4-block shared prefix diff --git a/tests/v1/engine/test_engine_args.py b/tests/v1/engine/test_engine_args.py index 5033f4768b..b1ba1adcfd 100644 --- a/tests/v1/engine/test_engine_args.py +++ b/tests/v1/engine/test_engine_args.py @@ -18,6 +18,7 @@ def test_prefix_caching_from_cli(): assert vllm_config.cache_config.enable_prefix_caching, ( "V1 turns on prefix caching by default." ) + assert vllm_config.cache_config.prefix_cache_retention_interval == 0 # Turn it off possible with flag. args = parser.parse_args(["--no-enable-prefix-caching"]) @@ -47,6 +48,10 @@ def test_prefix_caching_from_cli(): with pytest.raises(ArgumentError): args = parser.parse_args(["--prefix-caching-hash-algo", "invalid"]) + args = parser.parse_args(["--prefix-cache-retention-interval", "64"]) + vllm_config = EngineArgs.from_cli_args(args=args).create_engine_config() + assert vllm_config.cache_config.prefix_cache_retention_interval == 64 + @pytest.mark.skipif(_xxhash is None, reason="xxhash not installed") def test_prefix_caching_xxhash_from_cli(): diff --git a/tests/v1/engine/test_engine_core_client.py b/tests/v1/engine/test_engine_core_client.py index 5a943e8d60..74f7b803b8 100644 --- a/tests/v1/engine/test_engine_core_client.py +++ b/tests/v1/engine/test_engine_core_client.py @@ -1066,6 +1066,7 @@ def test_kv_cache_events( model=model_name, enforce_eager=True, enable_prefix_caching=True, + prefix_cache_retention_interval=None, block_size=block_size, ) engine_args.kv_events_config = publisher_config diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py index 4655c5716e..50d2ee1614 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py @@ -247,7 +247,9 @@ def _make_vllm_config( ) -def _make_kv_cache_config(*, block_size: int = 16) -> object: +def _make_kv_cache_config( + *, block_size: int = 16, prefix_cache_retention_interval: int | None = 0 +) -> object: """Minimal single-group KVCacheConfig for topology tests.""" from vllm.v1.kv_cache_interface import ( FullAttentionSpec, @@ -262,6 +264,7 @@ def _make_kv_cache_config(*, block_size: int = 16) -> object: num_blocks=10, kv_cache_tensors=[], kv_cache_groups=[KVCacheGroupSpec(["layer0"], spec)], + prefix_cache_retention_interval=prefix_cache_retention_interval, ) @@ -1594,6 +1597,7 @@ def test_requester_worker_init_uses_positional_setup(tmp_path, monkeypatch): "mlx5_0", "10.0.0.7:50051", ) + assert w.coord.retention_interval == 0 def test_requester_worker_init_prefers_local_hostname_override( diff --git a/vllm/config/cache.py b/vllm/config/cache.py index 0981f710ea..27e46534dc 100644 --- a/vllm/config/cache.py +++ b/vllm/config/cache.py @@ -7,7 +7,7 @@ from typing import Any, ClassVar, Literal from pydantic import Field, field_validator, model_validator -from vllm.config.utils import config +from vllm.config.utils import config, get_from_deprecated_env_if_set from vllm.logger import init_logger from vllm.utils.torch_utils import ( is_quantized_kv_cache, @@ -35,6 +35,17 @@ CacheDType = Literal[ "nvfp4", "nvfp4_4over6", ] + + +def _get_prefix_cache_retention_interval() -> int | None: + env_value = get_from_deprecated_env_if_set( + "VLLM_PREFIX_CACHE_RETENTION_INTERVAL", + "v0.29", + "prefix_cache_retention_interval", + ) + return 0 if env_value is None else int(env_value) + + MambaDType = Literal["auto", "float32", "float16", "bfloat16"] MambaCacheMode = Literal["all", "align", "none"] PrefixCachingHashAlgo = Literal["sha256", "sha256_cbor", "xxhash", "xxhash_cbor"] @@ -111,6 +122,15 @@ class CacheConfig: security risk tolerance against the performance benefits before turning this on. - "xxhash_cbor" combines canonical CBOR serialization with xxHash for reproducible hashing. Requires the optional ``xxhash`` package.""" + prefix_cache_retention_interval: int | None = Field( + default_factory=_get_prefix_cache_retention_interval, ge=0 + ) + """Token interval between retained sliding-window and Mamba prefix-cache + checkpoints. ``0`` retains only semantic checkpoints, including the latest + replay boundary and shared-prefix junctions. Positive values additionally + retain periodic checkpoints at the specified interval, which must be a + multiple of the scheduler block size. ``None`` retains checkpoints densely. + Applies only to sliding-window and Mamba cache groups.""" kv_cache_dtype_skip_layers: list[str] = field(default_factory=list) """Layer patterns to skip KV cache quantization. Accepts layer indices (e.g., '0', '2', '4') or attention type names (e.g., 'sliding_window').""" @@ -218,6 +238,7 @@ class CacheConfig: "num_gpu_blocks_override", "enable_prefix_caching", "prefix_caching_hash_algo", + "prefix_cache_retention_interval", # Prefix-caching implementation detail (doesn't affect compiled graph). "prefix_match_unit", "mamba_page_size_padded", diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py index e25aec3fbd..83c6d16e1b 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py @@ -1433,7 +1433,7 @@ class MooncakeStoreWorker: scheduler_block_size=self.block_size, hash_block_size=self.hash_block_size, use_eagle=use_eagle, - retention_interval=envs.VLLM_PREFIX_CACHE_RETENTION_INTERVAL, + retention_interval=kv_cache_config.prefix_cache_retention_interval, ) # One ChunkedTokenDatabase per group; addresses populated in # register_kv_caches once the kv-cache layout is known. Each group's diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 44ee837c6f..1fb1388f10 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -522,6 +522,9 @@ class EngineArgs: prefix_caching_hash_algo: PrefixCachingHashAlgo = ( CacheConfig.prefix_caching_hash_algo ) + prefix_cache_retention_interval: int | None = get_field( + CacheConfig, "prefix_cache_retention_interval" + ) disable_sliding_window: bool = ModelConfig.disable_sliding_window disable_cascade_attn: bool = ModelConfig.disable_cascade_attn offload_backend: str = OffloadConfig.offload_backend @@ -1224,6 +1227,10 @@ class EngineArgs: cache_group.add_argument( "--prefix-caching-hash-algo", **cache_kwargs["prefix_caching_hash_algo"] ) + cache_group.add_argument( + "--prefix-cache-retention-interval", + **cache_kwargs["prefix_cache_retention_interval"], + ) cache_group.add_argument( "--kv-cache-dtype-skip-layers", **cache_kwargs["kv_cache_dtype_skip_layers"] ) @@ -2001,6 +2008,7 @@ class EngineArgs: sliding_window=sliding_window, enable_prefix_caching=self.enable_prefix_caching, prefix_caching_hash_algo=self.prefix_caching_hash_algo, + prefix_cache_retention_interval=self.prefix_cache_retention_interval, kv_cache_dtype_skip_layers=self.kv_cache_dtype_skip_layers, kv_sharing_fast_prefill=self.kv_sharing_fast_prefill, mamba_cache_dtype=self.mamba_cache_dtype, diff --git a/vllm/envs.py b/vllm/envs.py index ed62659cfa..15705b7c17 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -1148,12 +1148,6 @@ environment_variables: dict[str, Callable[[], Any]] = { if "VLLM_PLUGINS" not in os.environ else os.environ["VLLM_PLUGINS"].split(",") ), - # Retain local sliding-window KV checkpoints for prefix caching. - # Unset (default) preserves the dense local checkpointing behavior. `0` - # retains only the latest completed prompt boundary. Positive values retain - # checkpoints at the specified interval boundaries (rounded up to the - # prefix-cache alignment). - # Applies to sliding-window attention for now but not yet Mamba/linear attention. "VLLM_PREFIX_CACHE_RETENTION_INTERVAL": lambda: ( int(os.environ["VLLM_PREFIX_CACHE_RETENTION_INTERVAL"]) if "VLLM_PREFIX_CACHE_RETENTION_INTERVAL" in os.environ diff --git a/vllm/v1/core/kv_cache_coordinator.py b/vllm/v1/core/kv_cache_coordinator.py index 8efaf9252e..0829ec0d2b 100644 --- a/vllm/v1/core/kv_cache_coordinator.py +++ b/vllm/v1/core/kv_cache_coordinator.py @@ -4,7 +4,6 @@ from abc import ABC, abstractmethod from collections.abc import Sequence from typing import NamedTuple -from vllm import envs from vllm.logger import init_logger from vllm.utils.math_utils import cdiv, round_down from vllm.v1.core.block_pool import BlockPool @@ -45,16 +44,18 @@ def _validate_prefix_cache_retention_interval( isinstance(g.kv_cache_spec, (SlidingWindowSpec, MambaSpec)) for g in kv_cache_config.kv_cache_groups ): + if retention_interval == 0: + return raise ValueError( - "VLLM_PREFIX_CACHE_RETENTION_INTERVAL is set but this model has " + "prefix_cache_retention_interval is set but this model has " "no sliding-window or Mamba KV cache group, so retention has no " - "effect. Unset it (it only applies to sliding-window and Mamba " + "effect. Set it to 0 (it only applies to sliding-window and Mamba " "attention)." ) if retention_interval < 0 or retention_interval % scheduler_block_size != 0: raise ValueError( - f"VLLM_PREFIX_CACHE_RETENTION_INTERVAL ({retention_interval}) " + f"prefix_cache_retention_interval ({retention_interval}) " "must be non-negative and a multiple of scheduler_block_size " f"({scheduler_block_size})." ) @@ -151,7 +152,7 @@ class KVCacheCoordinator(ABC): # A positive retention interval must be a multiple of the base hit granularity # (``scheduler_block_size``) to land on real cache-hit boundaries. # 0 = keep only the latest replay boundary; None = dense; - self.retention_interval = envs.VLLM_PREFIX_CACHE_RETENTION_INTERVAL + self.retention_interval = kv_cache_config.prefix_cache_retention_interval _validate_prefix_cache_retention_interval( self.retention_interval, self.scheduler_block_size, kv_cache_config ) diff --git a/vllm/v1/core/kv_cache_manager.py b/vllm/v1/core/kv_cache_manager.py index 44097c3da2..d1af91b659 100644 --- a/vllm/v1/core/kv_cache_manager.py +++ b/vllm/v1/core/kv_cache_manager.py @@ -243,7 +243,7 @@ class KVCacheManager: - ``shared_prefix_boundary``: the block-aligned token position of a shared prefix that a sparse-retention group (Mamba / sliding window) has not cached yet (Marconi-style APC), or 0 if none. - Pinned so ``VLLM_PREFIX_CACHE_RETENTION_INTERVAL`` does not drop + Pinned so sparse prefix-cache retention does not drop the junction and defeat cross-request reuse. """ # We skip finding the prefix cache hit when prefix caching is diff --git a/vllm/v1/core/kv_cache_utils.py b/vllm/v1/core/kv_cache_utils.py index ad51e4b2b3..d6e401d118 100644 --- a/vllm/v1/core/kv_cache_utils.py +++ b/vllm/v1/core/kv_cache_utils.py @@ -1347,6 +1347,9 @@ def get_kv_cache_config_from_groups( num_blocks=1, kv_cache_tensors=[], kv_cache_groups=kv_cache_groups, + prefix_cache_retention_interval=( + vllm_config.cache_config.prefix_cache_retention_interval + ), ) # Determine how model runners should initialize the KV cache tensors. @@ -1406,6 +1409,9 @@ def get_kv_cache_config_from_groups( num_blocks=num_blocks, kv_cache_tensors=kv_cache_tensors, kv_cache_groups=kv_cache_groups, + prefix_cache_retention_interval=( + vllm_config.cache_config.prefix_cache_retention_interval + ), ) diff --git a/vllm/v1/kv_cache_interface.py b/vllm/v1/kv_cache_interface.py index 90391f6ef0..cbead6d3c8 100644 --- a/vllm/v1/kv_cache_interface.py +++ b/vllm/v1/kv_cache_interface.py @@ -970,6 +970,8 @@ class KVCacheConfig: For models with multiple types of attention, there will be multiple groups, see `_get_kv_cache_config_uniform_page_size` for more details. """ + prefix_cache_retention_interval: int | None = None + """Resolved retention policy for local prefix-cache checkpoints.""" @property def has_mamba_layers(self) -> bool: diff --git a/vllm/v1/simple_kv_offload/manager.py b/vllm/v1/simple_kv_offload/manager.py index 2e6839fef5..1e4fcae68c 100644 --- a/vllm/v1/simple_kv_offload/manager.py +++ b/vllm/v1/simple_kv_offload/manager.py @@ -4,7 +4,7 @@ import contextlib from collections.abc import Iterable -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import TYPE_CHECKING, Any from vllm.config import VllmConfig @@ -190,7 +190,6 @@ class SimpleCPUOffloadScheduler: """Derive a CPU KVCacheConfig from the GPU config. Same kv_cache_groups, num_blocks scaled by CPU/GPU memory ratio.""" # Import here to avoid potential circular imports - from vllm.v1.kv_cache_interface import KVCacheConfig as KVCacheConfigCls from vllm.v1.kv_cache_interface import KVCacheTensor assert len(gpu_config.kv_cache_tensors) > 0 @@ -215,10 +214,10 @@ class SimpleCPUOffloadScheduler: for t in gpu_config.kv_cache_tensors ] - return KVCacheConfigCls( + return replace( + gpu_config, num_blocks=num_cpu_blocks, kv_cache_tensors=cpu_tensors, - kv_cache_groups=gpu_config.kv_cache_groups, ) @staticmethod