Promote prefix_cache_retention_interval to an argument and change the default to 0 (#52216)

Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Tyler Michael Smith
2026-08-17 13:20:50 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 1d3a8b9e22
commit 017e9f4448
17 changed files with 160 additions and 63 deletions
+5
View File
@@ -216,6 +216,11 @@ def test_cache_config_hash_ignores_kv_cache_sizing_knobs():
assert CacheConfig(gpu_memory_utilization=0.5).compute_hash() == base_hash
def test_cache_config_hash_ignores_prefix_cache_retention_interval():
base_hash = CacheConfig().compute_hash()
assert CacheConfig(prefix_cache_retention_interval=64).compute_hash() == base_hash
def test_envs_compile_factors_relocation_invariant(tmp_path):
"""Relocating HOME or the XDG roots must not change the compile-cache
env hash.
+23
View File
@@ -476,6 +476,7 @@ def test_prefix_cache_default():
# should be None by default (depends on model).
engine_args = EngineArgs.from_cli_args(args=args)
assert engine_args.enable_prefix_caching is None
assert engine_args.prefix_cache_retention_interval == 0
# with flag to turn it on.
args = parser.parse_args(["--enable-prefix-caching"])
@@ -487,6 +488,28 @@ def test_prefix_cache_default():
engine_args = EngineArgs.from_cli_args(args=args)
assert not engine_args.enable_prefix_caching
args = parser.parse_args(["--prefix-cache-retention-interval", "64"])
engine_args = EngineArgs.from_cli_args(args=args)
assert engine_args.prefix_cache_retention_interval == 64
def test_prefix_cache_retention_interval_from_deprecated_env(
monkeypatch, caplog, disable_log_dedup
):
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "64")
engine_args = EngineArgs()
assert engine_args.prefix_cache_retention_interval == 64
assert "VLLM_PREFIX_CACHE_RETENTION_INTERVAL" in caplog.text
assert "deprecated" in caplog.text
assert "prefix_cache_retention_interval" in caplog.text
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
args = parser.parse_args(["--prefix-cache-retention-interval", "32"])
engine_args = EngineArgs.from_cli_args(args)
assert engine_args.prefix_cache_retention_interval == 32
@pytest.mark.parametrize(
("arg", "expected", "option"),
@@ -88,6 +88,7 @@ def _make_groups(n_c4, n_c128, n_swa):
def _mock_vllm_config(kv_connector_extra_config: dict[str, str] | None = None):
config = MagicMock()
config.cache_config.num_gpu_blocks_override = None
config.cache_config.prefix_cache_retention_interval = 0
config.kv_transfer_config = None
if kv_connector_extra_config is not None:
config.kv_transfer_config = MagicMock()
@@ -322,6 +323,7 @@ class TestInterleavedPacking:
)
assert config.num_blocks == 32
assert config.prefix_cache_retention_interval == 0
assert sum(t.size for t in config.kv_cache_tensors) == page_size * 2 * 32
assert config.kv_cache_tensors == [
KVCacheTensor(size=page_size * 32, shared_by=["full.0", "sw.0", "sw.1"]),
+4
View File
@@ -816,6 +816,7 @@ def test_metrics_empty_stats():
def test_get_kv_cache_configs_multiple_workers():
model_config = ModelConfig(max_model_len=16)
vllm_config = VllmConfig(model_config=model_config)
vllm_config.cache_config.prefix_cache_retention_interval = None
ref_kv_cache_spec = new_kv_cache_spec()
same_kv_cache_specs = [
@@ -1173,6 +1174,7 @@ def test_get_kv_cache_configs_multiple_workers():
def test_get_kv_cache_configs_pp_sharding(asymmetric_memory):
model_config = ModelConfig(max_model_len=512)
vllm_config = VllmConfig(model_config=model_config)
vllm_config.cache_config.prefix_cache_retention_interval = None
ref_kv_cache_spec = new_kv_cache_spec()
pp_kv_cache_specs = [
@@ -1702,6 +1704,7 @@ def test_get_kv_cache_config_one_worker():
# pass max_model_len to pass check_enough_kv_cache_memory
model_config = ModelConfig(max_model_len=16)
vllm_config = VllmConfig(model_config=model_config)
vllm_config.cache_config.prefix_cache_retention_interval = None
mem_per_block_per_layer = 16 * 2 * 64 * 4 * 2
# all layers are full attention -> single group
@@ -2015,6 +2018,7 @@ def test_get_kv_cache_config_one_worker():
def test_get_kv_cache_configs_attention_free():
kv_cache_specs: dict[str, KVCacheSpec] = {}
vllm_config = VllmConfig(model_config=ModelConfig(max_model_len=16))
vllm_config.cache_config.prefix_cache_retention_interval = None
kv_cache_configs = get_kv_cache_configs(vllm_config, [kv_cache_specs], [0])
assert kv_cache_configs == [
KVCacheConfig(
+66 -44
View File
@@ -4,6 +4,7 @@
import copy
from collections.abc import Callable
from dataclasses import replace
from math import lcm
from types import SimpleNamespace
@@ -110,6 +111,11 @@ def make_kv_cache_manager(kv_cache_config: KVCacheConfig, **kwargs) -> KVCacheMa
"scheduler_block_size",
lcm(*(g.kv_cache_spec.block_size for g in kv_cache_config.kv_cache_groups)),
)
if "retention_interval" in kwargs:
kv_cache_config = replace(
kv_cache_config,
prefix_cache_retention_interval=kwargs.pop("retention_interval"),
)
return KVCacheManager(kv_cache_config, **kwargs)
@@ -3119,9 +3125,8 @@ def test_hybrid_cache_blocks_clamped_to_lcm():
)
def test_hybrid_local_kv_retention_interval_aligns_in_manager(monkeypatch):
def test_hybrid_local_kv_retention_interval_aligns_in_manager():
"""Verify fixed intervals retain sparse tails plus the latest replay tail."""
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "64")
block_size = 8
kv_cache_config = KVCacheConfig(
num_blocks=100,
@@ -3153,6 +3158,7 @@ def test_hybrid_local_kv_retention_interval_aligns_in_manager(monkeypatch):
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
retention_interval=64,
)
# The SWA manager uses the configured 64-token interval (a multiple of the
@@ -3184,18 +3190,15 @@ def test_hybrid_local_kv_retention_interval_aligns_in_manager(monkeypatch):
"interval, expected_match",
[
# scheduler_block_size is 32 (= lcm(4*8, 8)); 33 is not a multiple of it.
("33", "multiple of scheduler_block_size"),
(33, "multiple of scheduler_block_size"),
# A negative multiple (-32 % 32 == 0) must still be rejected explicitly,
# otherwise it would pass the modulo check and silently degrade to dense.
("-32", "non-negative"),
(-32, "non-negative"),
],
)
def test_hybrid_local_kv_retention_interval_rejects_invalid(
monkeypatch, interval, expected_match
):
def test_hybrid_local_kv_retention_interval_rejects_invalid(interval, expected_match):
"""A retention interval that is negative or not a multiple of
scheduler_block_size errors out at construction time."""
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", interval)
block_size = 8
kv_cache_config = KVCacheConfig(
num_blocks=100,
@@ -3228,12 +3231,36 @@ def test_hybrid_local_kv_retention_interval_rejects_invalid(
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
retention_interval=interval,
)
def test_hybrid_local_kv_retention_interval_survives_recycling(monkeypatch):
def test_zero_retention_is_ignored_for_full_attention():
kv_cache_config = make_kv_cache_config(block_size=16, num_blocks=10)
manager = make_kv_cache_manager(
kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=16,
retention_interval=0,
)
assert manager.coordinator.retention_interval == 0
def test_positive_retention_rejects_full_attention():
kv_cache_config = make_kv_cache_config(block_size=16, num_blocks=10)
with pytest.raises(ValueError, match="no sliding-window or Mamba"):
make_kv_cache_manager(
kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=16,
retention_interval=16,
)
def test_hybrid_local_kv_retention_interval_survives_recycling():
"""Verify retained local checkpoints are reused after block recycling."""
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "1024")
hash_block_size = 4
kv_cache_config = KVCacheConfig(
num_blocks=800,
@@ -3286,6 +3313,7 @@ def test_hybrid_local_kv_retention_interval_survives_recycling(monkeypatch):
max_model_len=4096,
enable_caching=True,
hash_block_size=hash_block_size,
retention_interval=1024,
)
def fill_request(request_id: str, token_offset: int) -> list[int]:
@@ -3314,9 +3342,8 @@ def test_hybrid_local_kv_retention_interval_survives_recycling(monkeypatch):
assert [len(blocks) for blocks in computed_blocks.blocks] == [4, 16, 128, 256]
def test_hybrid_local_kv_retention_latest_only_reuses_replay_boundary(monkeypatch):
def test_hybrid_local_kv_retention_latest_only_reuses_replay_boundary():
"""Verify latest-only retention reuses only the replayable prompt boundary."""
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "0")
block_size = 8
kv_cache_config = KVCacheConfig(
num_blocks=100,
@@ -3348,6 +3375,7 @@ def test_hybrid_local_kv_retention_latest_only_reuses_replay_boundary(monkeypatc
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
retention_interval=0,
)
token_ids = [i for i in range(16) for _ in range(block_size)]
@@ -3388,14 +3416,13 @@ def test_hybrid_local_kv_retention_latest_only_reuses_replay_boundary(monkeypatc
assert len(computed_blocks.blocks[1]) == 0
def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary(monkeypatch):
def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary():
"""Verify MTP/EAGLE SWA retention keeps the extra proof block.
EAGLE/MTP lookup matches one additional local block after the returned
prefix and then drops it. Sparse retention must therefore cache the normal
local tail at the latest replay boundary plus one extra SWA block.
"""
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "0")
block_size = 8
kv_cache_config = KVCacheConfig(
num_blocks=100,
@@ -3428,6 +3455,7 @@ def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary(monkeypatch):
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
retention_interval=0,
use_eagle=True,
)
@@ -3824,12 +3852,11 @@ def test_cache_hit_local_and_external_two_groups_preempt_and_reallocate():
assert manager.get_blocks("test").get_block_ids() != ([], [])
def test_swa_free_split_keeps_cached_tail_ahead_of_scratch(monkeypatch):
"""Default path (no retention): freeing an SWA request must place its
def test_swa_free_split_keeps_cached_tail_ahead_of_scratch():
"""Dense retention: freeing an SWA request must place its
uncached scratch blocks at the front of the free queue (recycled first)
and keep its cached checkpoint blocks at the back (retained for prefix
hits). This split is always-on, independent of the retention interval."""
monkeypatch.delenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", raising=False)
block_size = 8
kv_cache_config = KVCacheConfig(
num_blocks=100,
@@ -3937,13 +3964,14 @@ def _make_pure_swa_manager(block_size, sliding_window, num_blocks=100, **kwargs)
)
def test_pure_swa_retention_interval_caches_sparse_tails(monkeypatch):
def test_pure_swa_retention_interval_caches_sparse_tails():
"""Sparse retention must work for a pure-SWA single-group model, not just
hybrid models: only the per-interval tails plus the latest replay tail are
cached, and a replay still hits the latest replayable boundary."""
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "64")
block_size = 16
manager = _make_pure_swa_manager(block_size, sliding_window=block_size)
manager = _make_pure_swa_manager(
block_size, sliding_window=block_size, retention_interval=64
)
assert type(manager.coordinator).__name__ == "UnitaryKVCacheCoordinator"
token_ids = [i for i in range(16) for _ in range(block_size)]
@@ -3976,11 +4004,12 @@ def test_pure_swa_retention_interval_caches_sparse_tails(monkeypatch):
assert num_computed == 240
def test_pure_swa_retention_latest_only(monkeypatch):
def test_pure_swa_retention_latest_only():
"""`=0` on a pure-SWA model keeps only the latest replay tail."""
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "0")
block_size = 16
manager = _make_pure_swa_manager(block_size, sliding_window=block_size)
manager = _make_pure_swa_manager(
block_size, sliding_window=block_size, retention_interval=0
)
token_ids = [i for i in range(16) for _ in range(block_size)]
req = make_request("0", token_ids, block_size, sha256)
@@ -4008,10 +4037,9 @@ def test_pure_swa_retention_latest_only(monkeypatch):
assert num_computed == 240
def test_pure_swa_retention_dense_default_caches_all(monkeypatch):
"""With retention unset, a pure-SWA model must keep the dense behavior:
def test_pure_swa_dense_retention_caches_all():
"""With retention set to ``None``, a pure-SWA model keeps dense behavior:
every block boundary is a potential hit, so all blocks are cached."""
monkeypatch.delenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", raising=False)
block_size = 16
manager = _make_pure_swa_manager(block_size, sliding_window=block_size)
@@ -4037,7 +4065,7 @@ def test_pure_swa_retention_dense_default_caches_all(monkeypatch):
def test_mamba_reachable_block_mask_sparsifies_retention():
"""Mamba state-snapshot retention: with VLLM_PREFIX_CACHE_RETENTION_INTERVAL
"""Mamba state-snapshot retention: with a configured retention interval,
the manager keeps one cached state per interval-sized segment (plus the
latest replay boundary) instead of a snapshot per block, which is what
lets a small attention block_size avoid Mamba dominating the KV pool."""
@@ -4063,7 +4091,7 @@ def test_mamba_reachable_block_mask_sparsifies_retention():
)
return None if m is None else {i for i, v in enumerate(m) if v}
# Dense default (None) -> no mask, every block cached (unchanged behavior).
# Dense retention (None) -> no mask, every block cached.
assert retained(None) is None
# interval == block_size -> every block is a boundary -> stays dense.
assert retained(block_size) is None
@@ -4111,7 +4139,7 @@ def test_mamba_reachable_block_mask_pins_shared_prefix():
assert retained(0, 100) == {5, 14}
# Coexists with segment tails (interval 64 -> {3,7,11,15} + replay 14).
assert retained(64, 96) == {3, 5, 7, 11, 14, 15}
# Dense default ignores the hint (nothing to sparsify).
# Dense retention ignores the hint (nothing to sparsify).
assert retained(None, 96) is None
# Out-of-range boundary is a no-op (only replay 14 remains).
assert retained(0, 16 * block_size * 2) == {14}
@@ -4120,14 +4148,13 @@ def test_mamba_reachable_block_mask_pins_shared_prefix():
assert retained(0, None) == {14}
def test_mamba_shared_prefix_survives_zero_retention(monkeypatch):
def test_mamba_shared_prefix_survives_zero_retention():
"""Manager-level check of the full wiring: a pinned shared-prefix boundary
(``Request.shared_prefix_boundary``, set by the scheduler on Marconi-style
detection) keeps its Mamba state block cached under
``VLLM_PREFIX_CACHE_RETENTION_INTERVAL=0``, which otherwise retains only the
``prefix_cache_retention_interval=0``, which otherwise retains only the
end-of-prompt replay boundary. Without this, a shared prefix (junction
before ``num_prompt``) would be recomputed by every sharing request."""
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "0")
block_size = 16
# 16-block (256-token) prompt; replay boundary is block 240 // 16 - 1 = 14.
@@ -4140,6 +4167,7 @@ def test_mamba_shared_prefix_survives_zero_retention(monkeypatch):
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
retention_interval=0,
)
req = make_request("r", token_ids, block_size, sha256)
req.shared_prefix_boundary = shared_prefix_boundary
@@ -4163,24 +4191,21 @@ def test_mamba_shared_prefix_survives_zero_retention(monkeypatch):
assert cached_mamba_blocks(96) == {5, 14}
def test_mamba_shared_prefix_reuse_under_zero_retention(monkeypatch):
def test_mamba_shared_prefix_reuse_under_zero_retention():
"""Full cross-request Marconi flow: a partial shared prefix cached by the
detecting request must stay reusable by a later request under
``VLLM_PREFIX_CACHE_RETENTION_INTERVAL=0``. Without the pin the junction is
``prefix_cache_retention_interval=0``. Without the pin the junction is
masked out and the later request misses; with it (and under dense) the reuse
is preserved."""
block_size = 16
def last_req_hit(retention, pin):
if retention is None:
monkeypatch.delenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", raising=False)
else:
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", str(retention))
manager = make_kv_cache_manager(
_make_hybrid_kv_cache_config(block_size, 200, ["full", "mamba_align"]),
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
retention_interval=retention,
)
shared = [7 for _ in range(2 * block_size)] # 2-block shared prefix
@@ -4261,23 +4286,20 @@ def test_swa_reachable_block_mask_pins_shared_prefix():
assert retained(0, 0, block_size) == {14}
def test_swa_shared_prefix_reuse_under_zero_retention(monkeypatch):
def test_swa_shared_prefix_reuse_under_zero_retention():
"""SWA cross-request analog: a partial shared prefix's sliding-window tail
must stay reusable under ``VLLM_PREFIX_CACHE_RETENTION_INTERVAL=0``. Without
must stay reusable under ``prefix_cache_retention_interval=0``. Without
the pin the junction window is masked out and a later request misses; with
it (and under dense) reuse is preserved."""
block_size = 16
def last_req_hit(retention, pin):
if retention is None:
monkeypatch.delenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", raising=False)
else:
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", str(retention))
manager = make_kv_cache_manager(
_make_hybrid_kv_cache_config(block_size, 200, ["full", "sliding_window"]),
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
retention_interval=retention,
)
shared = [7 for _ in range(4 * block_size)] # 4-block shared prefix
+5
View File
@@ -18,6 +18,7 @@ def test_prefix_caching_from_cli():
assert vllm_config.cache_config.enable_prefix_caching, (
"V1 turns on prefix caching by default."
)
assert vllm_config.cache_config.prefix_cache_retention_interval == 0
# Turn it off possible with flag.
args = parser.parse_args(["--no-enable-prefix-caching"])
@@ -47,6 +48,10 @@ def test_prefix_caching_from_cli():
with pytest.raises(ArgumentError):
args = parser.parse_args(["--prefix-caching-hash-algo", "invalid"])
args = parser.parse_args(["--prefix-cache-retention-interval", "64"])
vllm_config = EngineArgs.from_cli_args(args=args).create_engine_config()
assert vllm_config.cache_config.prefix_cache_retention_interval == 64
@pytest.mark.skipif(_xxhash is None, reason="xxhash not installed")
def test_prefix_caching_xxhash_from_cli():
@@ -1066,6 +1066,7 @@ def test_kv_cache_events(
model=model_name,
enforce_eager=True,
enable_prefix_caching=True,
prefix_cache_retention_interval=None,
block_size=block_size,
)
engine_args.kv_events_config = publisher_config
@@ -247,7 +247,9 @@ def _make_vllm_config(
)
def _make_kv_cache_config(*, block_size: int = 16) -> object:
def _make_kv_cache_config(
*, block_size: int = 16, prefix_cache_retention_interval: int | None = 0
) -> object:
"""Minimal single-group KVCacheConfig for topology tests."""
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
@@ -262,6 +264,7 @@ def _make_kv_cache_config(*, block_size: int = 16) -> object:
num_blocks=10,
kv_cache_tensors=[],
kv_cache_groups=[KVCacheGroupSpec(["layer0"], spec)],
prefix_cache_retention_interval=prefix_cache_retention_interval,
)
@@ -1594,6 +1597,7 @@ def test_requester_worker_init_uses_positional_setup(tmp_path, monkeypatch):
"mlx5_0",
"10.0.0.7:50051",
)
assert w.coord.retention_interval == 0
def test_requester_worker_init_prefers_local_hostname_override(
+22 -1
View File
@@ -7,7 +7,7 @@ from typing import Any, ClassVar, Literal
from pydantic import Field, field_validator, model_validator
from vllm.config.utils import config
from vllm.config.utils import config, get_from_deprecated_env_if_set
from vllm.logger import init_logger
from vllm.utils.torch_utils import (
is_quantized_kv_cache,
@@ -35,6 +35,17 @@ CacheDType = Literal[
"nvfp4",
"nvfp4_4over6",
]
def _get_prefix_cache_retention_interval() -> int | None:
env_value = get_from_deprecated_env_if_set(
"VLLM_PREFIX_CACHE_RETENTION_INTERVAL",
"v0.29",
"prefix_cache_retention_interval",
)
return 0 if env_value is None else int(env_value)
MambaDType = Literal["auto", "float32", "float16", "bfloat16"]
MambaCacheMode = Literal["all", "align", "none"]
PrefixCachingHashAlgo = Literal["sha256", "sha256_cbor", "xxhash", "xxhash_cbor"]
@@ -111,6 +122,15 @@ class CacheConfig:
security risk tolerance against the performance benefits before turning this on.
- "xxhash_cbor" combines canonical CBOR serialization with xxHash for
reproducible hashing. Requires the optional ``xxhash`` package."""
prefix_cache_retention_interval: int | None = Field(
default_factory=_get_prefix_cache_retention_interval, ge=0
)
"""Token interval between retained sliding-window and Mamba prefix-cache
checkpoints. ``0`` retains only semantic checkpoints, including the latest
replay boundary and shared-prefix junctions. Positive values additionally
retain periodic checkpoints at the specified interval, which must be a
multiple of the scheduler block size. ``None`` retains checkpoints densely.
Applies only to sliding-window and Mamba cache groups."""
kv_cache_dtype_skip_layers: list[str] = field(default_factory=list)
"""Layer patterns to skip KV cache quantization. Accepts layer indices
(e.g., '0', '2', '4') or attention type names (e.g., 'sliding_window')."""
@@ -218,6 +238,7 @@ class CacheConfig:
"num_gpu_blocks_override",
"enable_prefix_caching",
"prefix_caching_hash_algo",
"prefix_cache_retention_interval",
# Prefix-caching implementation detail (doesn't affect compiled graph).
"prefix_match_unit",
"mamba_page_size_padded",
@@ -1433,7 +1433,7 @@ class MooncakeStoreWorker:
scheduler_block_size=self.block_size,
hash_block_size=self.hash_block_size,
use_eagle=use_eagle,
retention_interval=envs.VLLM_PREFIX_CACHE_RETENTION_INTERVAL,
retention_interval=kv_cache_config.prefix_cache_retention_interval,
)
# One ChunkedTokenDatabase per group; addresses populated in
# register_kv_caches once the kv-cache layout is known. Each group's
+8
View File
@@ -522,6 +522,9 @@ class EngineArgs:
prefix_caching_hash_algo: PrefixCachingHashAlgo = (
CacheConfig.prefix_caching_hash_algo
)
prefix_cache_retention_interval: int | None = get_field(
CacheConfig, "prefix_cache_retention_interval"
)
disable_sliding_window: bool = ModelConfig.disable_sliding_window
disable_cascade_attn: bool = ModelConfig.disable_cascade_attn
offload_backend: str = OffloadConfig.offload_backend
@@ -1224,6 +1227,10 @@ class EngineArgs:
cache_group.add_argument(
"--prefix-caching-hash-algo", **cache_kwargs["prefix_caching_hash_algo"]
)
cache_group.add_argument(
"--prefix-cache-retention-interval",
**cache_kwargs["prefix_cache_retention_interval"],
)
cache_group.add_argument(
"--kv-cache-dtype-skip-layers", **cache_kwargs["kv_cache_dtype_skip_layers"]
)
@@ -2001,6 +2008,7 @@ class EngineArgs:
sliding_window=sliding_window,
enable_prefix_caching=self.enable_prefix_caching,
prefix_caching_hash_algo=self.prefix_caching_hash_algo,
prefix_cache_retention_interval=self.prefix_cache_retention_interval,
kv_cache_dtype_skip_layers=self.kv_cache_dtype_skip_layers,
kv_sharing_fast_prefill=self.kv_sharing_fast_prefill,
mamba_cache_dtype=self.mamba_cache_dtype,
-6
View File
@@ -1148,12 +1148,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
if "VLLM_PLUGINS" not in os.environ
else os.environ["VLLM_PLUGINS"].split(",")
),
# Retain local sliding-window KV checkpoints for prefix caching.
# Unset (default) preserves the dense local checkpointing behavior. `0`
# retains only the latest completed prompt boundary. Positive values retain
# checkpoints at the specified interval boundaries (rounded up to the
# prefix-cache alignment).
# Applies to sliding-window attention for now but not yet Mamba/linear attention.
"VLLM_PREFIX_CACHE_RETENTION_INTERVAL": lambda: (
int(os.environ["VLLM_PREFIX_CACHE_RETENTION_INTERVAL"])
if "VLLM_PREFIX_CACHE_RETENTION_INTERVAL" in os.environ
+6 -5
View File
@@ -4,7 +4,6 @@ from abc import ABC, abstractmethod
from collections.abc import Sequence
from typing import NamedTuple
from vllm import envs
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv, round_down
from vllm.v1.core.block_pool import BlockPool
@@ -45,16 +44,18 @@ def _validate_prefix_cache_retention_interval(
isinstance(g.kv_cache_spec, (SlidingWindowSpec, MambaSpec))
for g in kv_cache_config.kv_cache_groups
):
if retention_interval == 0:
return
raise ValueError(
"VLLM_PREFIX_CACHE_RETENTION_INTERVAL is set but this model has "
"prefix_cache_retention_interval is set but this model has "
"no sliding-window or Mamba KV cache group, so retention has no "
"effect. Unset it (it only applies to sliding-window and Mamba "
"effect. Set it to 0 (it only applies to sliding-window and Mamba "
"attention)."
)
if retention_interval < 0 or retention_interval % scheduler_block_size != 0:
raise ValueError(
f"VLLM_PREFIX_CACHE_RETENTION_INTERVAL ({retention_interval}) "
f"prefix_cache_retention_interval ({retention_interval}) "
"must be non-negative and a multiple of scheduler_block_size "
f"({scheduler_block_size})."
)
@@ -151,7 +152,7 @@ class KVCacheCoordinator(ABC):
# A positive retention interval must be a multiple of the base hit granularity
# (``scheduler_block_size``) to land on real cache-hit boundaries.
# 0 = keep only the latest replay boundary; None = dense;
self.retention_interval = envs.VLLM_PREFIX_CACHE_RETENTION_INTERVAL
self.retention_interval = kv_cache_config.prefix_cache_retention_interval
_validate_prefix_cache_retention_interval(
self.retention_interval, self.scheduler_block_size, kv_cache_config
)
+1 -1
View File
@@ -243,7 +243,7 @@ class KVCacheManager:
- ``shared_prefix_boundary``: the block-aligned token position of
a shared prefix that a sparse-retention group (Mamba / sliding
window) has not cached yet (Marconi-style APC), or 0 if none.
Pinned so ``VLLM_PREFIX_CACHE_RETENTION_INTERVAL`` does not drop
Pinned so sparse prefix-cache retention does not drop
the junction and defeat cross-request reuse.
"""
# We skip finding the prefix cache hit when prefix caching is
+6
View File
@@ -1347,6 +1347,9 @@ def get_kv_cache_config_from_groups(
num_blocks=1,
kv_cache_tensors=[],
kv_cache_groups=kv_cache_groups,
prefix_cache_retention_interval=(
vllm_config.cache_config.prefix_cache_retention_interval
),
)
# Determine how model runners should initialize the KV cache tensors.
@@ -1406,6 +1409,9 @@ def get_kv_cache_config_from_groups(
num_blocks=num_blocks,
kv_cache_tensors=kv_cache_tensors,
kv_cache_groups=kv_cache_groups,
prefix_cache_retention_interval=(
vllm_config.cache_config.prefix_cache_retention_interval
),
)
+2
View File
@@ -970,6 +970,8 @@ class KVCacheConfig:
For models with multiple types of attention, there will be multiple groups,
see `_get_kv_cache_config_uniform_page_size` for more details.
"""
prefix_cache_retention_interval: int | None = None
"""Resolved retention policy for local prefix-cache checkpoints."""
@property
def has_mamba_layers(self) -> bool:
+3 -4
View File
@@ -4,7 +4,7 @@
import contextlib
from collections.abc import Iterable
from dataclasses import dataclass, field
from dataclasses import dataclass, field, replace
from typing import TYPE_CHECKING, Any
from vllm.config import VllmConfig
@@ -190,7 +190,6 @@ class SimpleCPUOffloadScheduler:
"""Derive a CPU KVCacheConfig from the GPU config.
Same kv_cache_groups, num_blocks scaled by CPU/GPU memory ratio."""
# Import here to avoid potential circular imports
from vllm.v1.kv_cache_interface import KVCacheConfig as KVCacheConfigCls
from vllm.v1.kv_cache_interface import KVCacheTensor
assert len(gpu_config.kv_cache_tensors) > 0
@@ -215,10 +214,10 @@ class SimpleCPUOffloadScheduler:
for t in gpu_config.kv_cache_tensors
]
return KVCacheConfigCls(
return replace(
gpu_config,
num_blocks=num_cpu_blocks,
kv_cache_tensors=cpu_tensors,
kv_cache_groups=gpu_config.kv_cache_groups,
)
@staticmethod