Promote prefix_cache_retention_interval to an argument and change the default to 0 (#52216)
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
1d3a8b9e22
commit
017e9f4448
@@ -216,6 +216,11 @@ def test_cache_config_hash_ignores_kv_cache_sizing_knobs():
|
||||
assert CacheConfig(gpu_memory_utilization=0.5).compute_hash() == base_hash
|
||||
|
||||
|
||||
def test_cache_config_hash_ignores_prefix_cache_retention_interval():
|
||||
base_hash = CacheConfig().compute_hash()
|
||||
assert CacheConfig(prefix_cache_retention_interval=64).compute_hash() == base_hash
|
||||
|
||||
|
||||
def test_envs_compile_factors_relocation_invariant(tmp_path):
|
||||
"""Relocating HOME or the XDG roots must not change the compile-cache
|
||||
env hash.
|
||||
|
||||
@@ -476,6 +476,7 @@ def test_prefix_cache_default():
|
||||
# should be None by default (depends on model).
|
||||
engine_args = EngineArgs.from_cli_args(args=args)
|
||||
assert engine_args.enable_prefix_caching is None
|
||||
assert engine_args.prefix_cache_retention_interval == 0
|
||||
|
||||
# with flag to turn it on.
|
||||
args = parser.parse_args(["--enable-prefix-caching"])
|
||||
@@ -487,6 +488,28 @@ def test_prefix_cache_default():
|
||||
engine_args = EngineArgs.from_cli_args(args=args)
|
||||
assert not engine_args.enable_prefix_caching
|
||||
|
||||
args = parser.parse_args(["--prefix-cache-retention-interval", "64"])
|
||||
engine_args = EngineArgs.from_cli_args(args=args)
|
||||
assert engine_args.prefix_cache_retention_interval == 64
|
||||
|
||||
|
||||
def test_prefix_cache_retention_interval_from_deprecated_env(
|
||||
monkeypatch, caplog, disable_log_dedup
|
||||
):
|
||||
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "64")
|
||||
|
||||
engine_args = EngineArgs()
|
||||
|
||||
assert engine_args.prefix_cache_retention_interval == 64
|
||||
assert "VLLM_PREFIX_CACHE_RETENTION_INTERVAL" in caplog.text
|
||||
assert "deprecated" in caplog.text
|
||||
assert "prefix_cache_retention_interval" in caplog.text
|
||||
|
||||
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
|
||||
args = parser.parse_args(["--prefix-cache-retention-interval", "32"])
|
||||
engine_args = EngineArgs.from_cli_args(args)
|
||||
assert engine_args.prefix_cache_retention_interval == 32
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("arg", "expected", "option"),
|
||||
|
||||
@@ -88,6 +88,7 @@ def _make_groups(n_c4, n_c128, n_swa):
|
||||
def _mock_vllm_config(kv_connector_extra_config: dict[str, str] | None = None):
|
||||
config = MagicMock()
|
||||
config.cache_config.num_gpu_blocks_override = None
|
||||
config.cache_config.prefix_cache_retention_interval = 0
|
||||
config.kv_transfer_config = None
|
||||
if kv_connector_extra_config is not None:
|
||||
config.kv_transfer_config = MagicMock()
|
||||
@@ -322,6 +323,7 @@ class TestInterleavedPacking:
|
||||
)
|
||||
|
||||
assert config.num_blocks == 32
|
||||
assert config.prefix_cache_retention_interval == 0
|
||||
assert sum(t.size for t in config.kv_cache_tensors) == page_size * 2 * 32
|
||||
assert config.kv_cache_tensors == [
|
||||
KVCacheTensor(size=page_size * 32, shared_by=["full.0", "sw.0", "sw.1"]),
|
||||
|
||||
@@ -816,6 +816,7 @@ def test_metrics_empty_stats():
|
||||
def test_get_kv_cache_configs_multiple_workers():
|
||||
model_config = ModelConfig(max_model_len=16)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
vllm_config.cache_config.prefix_cache_retention_interval = None
|
||||
|
||||
ref_kv_cache_spec = new_kv_cache_spec()
|
||||
same_kv_cache_specs = [
|
||||
@@ -1173,6 +1174,7 @@ def test_get_kv_cache_configs_multiple_workers():
|
||||
def test_get_kv_cache_configs_pp_sharding(asymmetric_memory):
|
||||
model_config = ModelConfig(max_model_len=512)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
vllm_config.cache_config.prefix_cache_retention_interval = None
|
||||
|
||||
ref_kv_cache_spec = new_kv_cache_spec()
|
||||
pp_kv_cache_specs = [
|
||||
@@ -1702,6 +1704,7 @@ def test_get_kv_cache_config_one_worker():
|
||||
# pass max_model_len to pass check_enough_kv_cache_memory
|
||||
model_config = ModelConfig(max_model_len=16)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
vllm_config.cache_config.prefix_cache_retention_interval = None
|
||||
|
||||
mem_per_block_per_layer = 16 * 2 * 64 * 4 * 2
|
||||
# all layers are full attention -> single group
|
||||
@@ -2015,6 +2018,7 @@ def test_get_kv_cache_config_one_worker():
|
||||
def test_get_kv_cache_configs_attention_free():
|
||||
kv_cache_specs: dict[str, KVCacheSpec] = {}
|
||||
vllm_config = VllmConfig(model_config=ModelConfig(max_model_len=16))
|
||||
vllm_config.cache_config.prefix_cache_retention_interval = None
|
||||
kv_cache_configs = get_kv_cache_configs(vllm_config, [kv_cache_specs], [0])
|
||||
assert kv_cache_configs == [
|
||||
KVCacheConfig(
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
import copy
|
||||
from collections.abc import Callable
|
||||
from dataclasses import replace
|
||||
from math import lcm
|
||||
from types import SimpleNamespace
|
||||
|
||||
@@ -110,6 +111,11 @@ def make_kv_cache_manager(kv_cache_config: KVCacheConfig, **kwargs) -> KVCacheMa
|
||||
"scheduler_block_size",
|
||||
lcm(*(g.kv_cache_spec.block_size for g in kv_cache_config.kv_cache_groups)),
|
||||
)
|
||||
if "retention_interval" in kwargs:
|
||||
kv_cache_config = replace(
|
||||
kv_cache_config,
|
||||
prefix_cache_retention_interval=kwargs.pop("retention_interval"),
|
||||
)
|
||||
return KVCacheManager(kv_cache_config, **kwargs)
|
||||
|
||||
|
||||
@@ -3119,9 +3125,8 @@ def test_hybrid_cache_blocks_clamped_to_lcm():
|
||||
)
|
||||
|
||||
|
||||
def test_hybrid_local_kv_retention_interval_aligns_in_manager(monkeypatch):
|
||||
def test_hybrid_local_kv_retention_interval_aligns_in_manager():
|
||||
"""Verify fixed intervals retain sparse tails plus the latest replay tail."""
|
||||
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "64")
|
||||
block_size = 8
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=100,
|
||||
@@ -3153,6 +3158,7 @@ def test_hybrid_local_kv_retention_interval_aligns_in_manager(monkeypatch):
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
retention_interval=64,
|
||||
)
|
||||
|
||||
# The SWA manager uses the configured 64-token interval (a multiple of the
|
||||
@@ -3184,18 +3190,15 @@ def test_hybrid_local_kv_retention_interval_aligns_in_manager(monkeypatch):
|
||||
"interval, expected_match",
|
||||
[
|
||||
# scheduler_block_size is 32 (= lcm(4*8, 8)); 33 is not a multiple of it.
|
||||
("33", "multiple of scheduler_block_size"),
|
||||
(33, "multiple of scheduler_block_size"),
|
||||
# A negative multiple (-32 % 32 == 0) must still be rejected explicitly,
|
||||
# otherwise it would pass the modulo check and silently degrade to dense.
|
||||
("-32", "non-negative"),
|
||||
(-32, "non-negative"),
|
||||
],
|
||||
)
|
||||
def test_hybrid_local_kv_retention_interval_rejects_invalid(
|
||||
monkeypatch, interval, expected_match
|
||||
):
|
||||
def test_hybrid_local_kv_retention_interval_rejects_invalid(interval, expected_match):
|
||||
"""A retention interval that is negative or not a multiple of
|
||||
scheduler_block_size errors out at construction time."""
|
||||
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", interval)
|
||||
block_size = 8
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=100,
|
||||
@@ -3228,12 +3231,36 @@ def test_hybrid_local_kv_retention_interval_rejects_invalid(
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
retention_interval=interval,
|
||||
)
|
||||
|
||||
|
||||
def test_hybrid_local_kv_retention_interval_survives_recycling(monkeypatch):
|
||||
def test_zero_retention_is_ignored_for_full_attention():
|
||||
kv_cache_config = make_kv_cache_config(block_size=16, num_blocks=10)
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=16,
|
||||
retention_interval=0,
|
||||
)
|
||||
assert manager.coordinator.retention_interval == 0
|
||||
|
||||
|
||||
def test_positive_retention_rejects_full_attention():
|
||||
kv_cache_config = make_kv_cache_config(block_size=16, num_blocks=10)
|
||||
with pytest.raises(ValueError, match="no sliding-window or Mamba"):
|
||||
make_kv_cache_manager(
|
||||
kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=16,
|
||||
retention_interval=16,
|
||||
)
|
||||
|
||||
|
||||
def test_hybrid_local_kv_retention_interval_survives_recycling():
|
||||
"""Verify retained local checkpoints are reused after block recycling."""
|
||||
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "1024")
|
||||
hash_block_size = 4
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=800,
|
||||
@@ -3286,6 +3313,7 @@ def test_hybrid_local_kv_retention_interval_survives_recycling(monkeypatch):
|
||||
max_model_len=4096,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
retention_interval=1024,
|
||||
)
|
||||
|
||||
def fill_request(request_id: str, token_offset: int) -> list[int]:
|
||||
@@ -3314,9 +3342,8 @@ def test_hybrid_local_kv_retention_interval_survives_recycling(monkeypatch):
|
||||
assert [len(blocks) for blocks in computed_blocks.blocks] == [4, 16, 128, 256]
|
||||
|
||||
|
||||
def test_hybrid_local_kv_retention_latest_only_reuses_replay_boundary(monkeypatch):
|
||||
def test_hybrid_local_kv_retention_latest_only_reuses_replay_boundary():
|
||||
"""Verify latest-only retention reuses only the replayable prompt boundary."""
|
||||
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "0")
|
||||
block_size = 8
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=100,
|
||||
@@ -3348,6 +3375,7 @@ def test_hybrid_local_kv_retention_latest_only_reuses_replay_boundary(monkeypatc
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
retention_interval=0,
|
||||
)
|
||||
|
||||
token_ids = [i for i in range(16) for _ in range(block_size)]
|
||||
@@ -3388,14 +3416,13 @@ def test_hybrid_local_kv_retention_latest_only_reuses_replay_boundary(monkeypatc
|
||||
assert len(computed_blocks.blocks[1]) == 0
|
||||
|
||||
|
||||
def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary(monkeypatch):
|
||||
def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary():
|
||||
"""Verify MTP/EAGLE SWA retention keeps the extra proof block.
|
||||
|
||||
EAGLE/MTP lookup matches one additional local block after the returned
|
||||
prefix and then drops it. Sparse retention must therefore cache the normal
|
||||
local tail at the latest replay boundary plus one extra SWA block.
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "0")
|
||||
block_size = 8
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=100,
|
||||
@@ -3428,6 +3455,7 @@ def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary(monkeypatch):
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
retention_interval=0,
|
||||
use_eagle=True,
|
||||
)
|
||||
|
||||
@@ -3824,12 +3852,11 @@ def test_cache_hit_local_and_external_two_groups_preempt_and_reallocate():
|
||||
assert manager.get_blocks("test").get_block_ids() != ([], [])
|
||||
|
||||
|
||||
def test_swa_free_split_keeps_cached_tail_ahead_of_scratch(monkeypatch):
|
||||
"""Default path (no retention): freeing an SWA request must place its
|
||||
def test_swa_free_split_keeps_cached_tail_ahead_of_scratch():
|
||||
"""Dense retention: freeing an SWA request must place its
|
||||
uncached scratch blocks at the front of the free queue (recycled first)
|
||||
and keep its cached checkpoint blocks at the back (retained for prefix
|
||||
hits). This split is always-on, independent of the retention interval."""
|
||||
monkeypatch.delenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", raising=False)
|
||||
block_size = 8
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=100,
|
||||
@@ -3937,13 +3964,14 @@ def _make_pure_swa_manager(block_size, sliding_window, num_blocks=100, **kwargs)
|
||||
)
|
||||
|
||||
|
||||
def test_pure_swa_retention_interval_caches_sparse_tails(monkeypatch):
|
||||
def test_pure_swa_retention_interval_caches_sparse_tails():
|
||||
"""Sparse retention must work for a pure-SWA single-group model, not just
|
||||
hybrid models: only the per-interval tails plus the latest replay tail are
|
||||
cached, and a replay still hits the latest replayable boundary."""
|
||||
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "64")
|
||||
block_size = 16
|
||||
manager = _make_pure_swa_manager(block_size, sliding_window=block_size)
|
||||
manager = _make_pure_swa_manager(
|
||||
block_size, sliding_window=block_size, retention_interval=64
|
||||
)
|
||||
assert type(manager.coordinator).__name__ == "UnitaryKVCacheCoordinator"
|
||||
|
||||
token_ids = [i for i in range(16) for _ in range(block_size)]
|
||||
@@ -3976,11 +4004,12 @@ def test_pure_swa_retention_interval_caches_sparse_tails(monkeypatch):
|
||||
assert num_computed == 240
|
||||
|
||||
|
||||
def test_pure_swa_retention_latest_only(monkeypatch):
|
||||
def test_pure_swa_retention_latest_only():
|
||||
"""`=0` on a pure-SWA model keeps only the latest replay tail."""
|
||||
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "0")
|
||||
block_size = 16
|
||||
manager = _make_pure_swa_manager(block_size, sliding_window=block_size)
|
||||
manager = _make_pure_swa_manager(
|
||||
block_size, sliding_window=block_size, retention_interval=0
|
||||
)
|
||||
|
||||
token_ids = [i for i in range(16) for _ in range(block_size)]
|
||||
req = make_request("0", token_ids, block_size, sha256)
|
||||
@@ -4008,10 +4037,9 @@ def test_pure_swa_retention_latest_only(monkeypatch):
|
||||
assert num_computed == 240
|
||||
|
||||
|
||||
def test_pure_swa_retention_dense_default_caches_all(monkeypatch):
|
||||
"""With retention unset, a pure-SWA model must keep the dense behavior:
|
||||
def test_pure_swa_dense_retention_caches_all():
|
||||
"""With retention set to ``None``, a pure-SWA model keeps dense behavior:
|
||||
every block boundary is a potential hit, so all blocks are cached."""
|
||||
monkeypatch.delenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", raising=False)
|
||||
block_size = 16
|
||||
manager = _make_pure_swa_manager(block_size, sliding_window=block_size)
|
||||
|
||||
@@ -4037,7 +4065,7 @@ def test_pure_swa_retention_dense_default_caches_all(monkeypatch):
|
||||
|
||||
|
||||
def test_mamba_reachable_block_mask_sparsifies_retention():
|
||||
"""Mamba state-snapshot retention: with VLLM_PREFIX_CACHE_RETENTION_INTERVAL
|
||||
"""Mamba state-snapshot retention: with a configured retention interval,
|
||||
the manager keeps one cached state per interval-sized segment (plus the
|
||||
latest replay boundary) instead of a snapshot per block, which is what
|
||||
lets a small attention block_size avoid Mamba dominating the KV pool."""
|
||||
@@ -4063,7 +4091,7 @@ def test_mamba_reachable_block_mask_sparsifies_retention():
|
||||
)
|
||||
return None if m is None else {i for i, v in enumerate(m) if v}
|
||||
|
||||
# Dense default (None) -> no mask, every block cached (unchanged behavior).
|
||||
# Dense retention (None) -> no mask, every block cached.
|
||||
assert retained(None) is None
|
||||
# interval == block_size -> every block is a boundary -> stays dense.
|
||||
assert retained(block_size) is None
|
||||
@@ -4111,7 +4139,7 @@ def test_mamba_reachable_block_mask_pins_shared_prefix():
|
||||
assert retained(0, 100) == {5, 14}
|
||||
# Coexists with segment tails (interval 64 -> {3,7,11,15} + replay 14).
|
||||
assert retained(64, 96) == {3, 5, 7, 11, 14, 15}
|
||||
# Dense default ignores the hint (nothing to sparsify).
|
||||
# Dense retention ignores the hint (nothing to sparsify).
|
||||
assert retained(None, 96) is None
|
||||
# Out-of-range boundary is a no-op (only replay 14 remains).
|
||||
assert retained(0, 16 * block_size * 2) == {14}
|
||||
@@ -4120,14 +4148,13 @@ def test_mamba_reachable_block_mask_pins_shared_prefix():
|
||||
assert retained(0, None) == {14}
|
||||
|
||||
|
||||
def test_mamba_shared_prefix_survives_zero_retention(monkeypatch):
|
||||
def test_mamba_shared_prefix_survives_zero_retention():
|
||||
"""Manager-level check of the full wiring: a pinned shared-prefix boundary
|
||||
(``Request.shared_prefix_boundary``, set by the scheduler on Marconi-style
|
||||
detection) keeps its Mamba state block cached under
|
||||
``VLLM_PREFIX_CACHE_RETENTION_INTERVAL=0``, which otherwise retains only the
|
||||
``prefix_cache_retention_interval=0``, which otherwise retains only the
|
||||
end-of-prompt replay boundary. Without this, a shared prefix (junction
|
||||
before ``num_prompt``) would be recomputed by every sharing request."""
|
||||
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "0")
|
||||
block_size = 16
|
||||
|
||||
# 16-block (256-token) prompt; replay boundary is block 240 // 16 - 1 = 14.
|
||||
@@ -4140,6 +4167,7 @@ def test_mamba_shared_prefix_survives_zero_retention(monkeypatch):
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
retention_interval=0,
|
||||
)
|
||||
req = make_request("r", token_ids, block_size, sha256)
|
||||
req.shared_prefix_boundary = shared_prefix_boundary
|
||||
@@ -4163,24 +4191,21 @@ def test_mamba_shared_prefix_survives_zero_retention(monkeypatch):
|
||||
assert cached_mamba_blocks(96) == {5, 14}
|
||||
|
||||
|
||||
def test_mamba_shared_prefix_reuse_under_zero_retention(monkeypatch):
|
||||
def test_mamba_shared_prefix_reuse_under_zero_retention():
|
||||
"""Full cross-request Marconi flow: a partial shared prefix cached by the
|
||||
detecting request must stay reusable by a later request under
|
||||
``VLLM_PREFIX_CACHE_RETENTION_INTERVAL=0``. Without the pin the junction is
|
||||
``prefix_cache_retention_interval=0``. Without the pin the junction is
|
||||
masked out and the later request misses; with it (and under dense) the reuse
|
||||
is preserved."""
|
||||
block_size = 16
|
||||
|
||||
def last_req_hit(retention, pin):
|
||||
if retention is None:
|
||||
monkeypatch.delenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", raising=False)
|
||||
else:
|
||||
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", str(retention))
|
||||
manager = make_kv_cache_manager(
|
||||
_make_hybrid_kv_cache_config(block_size, 200, ["full", "mamba_align"]),
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
retention_interval=retention,
|
||||
)
|
||||
shared = [7 for _ in range(2 * block_size)] # 2-block shared prefix
|
||||
|
||||
@@ -4261,23 +4286,20 @@ def test_swa_reachable_block_mask_pins_shared_prefix():
|
||||
assert retained(0, 0, block_size) == {14}
|
||||
|
||||
|
||||
def test_swa_shared_prefix_reuse_under_zero_retention(monkeypatch):
|
||||
def test_swa_shared_prefix_reuse_under_zero_retention():
|
||||
"""SWA cross-request analog: a partial shared prefix's sliding-window tail
|
||||
must stay reusable under ``VLLM_PREFIX_CACHE_RETENTION_INTERVAL=0``. Without
|
||||
must stay reusable under ``prefix_cache_retention_interval=0``. Without
|
||||
the pin the junction window is masked out and a later request misses; with
|
||||
it (and under dense) reuse is preserved."""
|
||||
block_size = 16
|
||||
|
||||
def last_req_hit(retention, pin):
|
||||
if retention is None:
|
||||
monkeypatch.delenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", raising=False)
|
||||
else:
|
||||
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", str(retention))
|
||||
manager = make_kv_cache_manager(
|
||||
_make_hybrid_kv_cache_config(block_size, 200, ["full", "sliding_window"]),
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
retention_interval=retention,
|
||||
)
|
||||
shared = [7 for _ in range(4 * block_size)] # 4-block shared prefix
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ def test_prefix_caching_from_cli():
|
||||
assert vllm_config.cache_config.enable_prefix_caching, (
|
||||
"V1 turns on prefix caching by default."
|
||||
)
|
||||
assert vllm_config.cache_config.prefix_cache_retention_interval == 0
|
||||
|
||||
# Turn it off possible with flag.
|
||||
args = parser.parse_args(["--no-enable-prefix-caching"])
|
||||
@@ -47,6 +48,10 @@ def test_prefix_caching_from_cli():
|
||||
with pytest.raises(ArgumentError):
|
||||
args = parser.parse_args(["--prefix-caching-hash-algo", "invalid"])
|
||||
|
||||
args = parser.parse_args(["--prefix-cache-retention-interval", "64"])
|
||||
vllm_config = EngineArgs.from_cli_args(args=args).create_engine_config()
|
||||
assert vllm_config.cache_config.prefix_cache_retention_interval == 64
|
||||
|
||||
|
||||
@pytest.mark.skipif(_xxhash is None, reason="xxhash not installed")
|
||||
def test_prefix_caching_xxhash_from_cli():
|
||||
|
||||
@@ -1066,6 +1066,7 @@ def test_kv_cache_events(
|
||||
model=model_name,
|
||||
enforce_eager=True,
|
||||
enable_prefix_caching=True,
|
||||
prefix_cache_retention_interval=None,
|
||||
block_size=block_size,
|
||||
)
|
||||
engine_args.kv_events_config = publisher_config
|
||||
|
||||
@@ -247,7 +247,9 @@ def _make_vllm_config(
|
||||
)
|
||||
|
||||
|
||||
def _make_kv_cache_config(*, block_size: int = 16) -> object:
|
||||
def _make_kv_cache_config(
|
||||
*, block_size: int = 16, prefix_cache_retention_interval: int | None = 0
|
||||
) -> object:
|
||||
"""Minimal single-group KVCacheConfig for topology tests."""
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
@@ -262,6 +264,7 @@ def _make_kv_cache_config(*, block_size: int = 16) -> object:
|
||||
num_blocks=10,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["layer0"], spec)],
|
||||
prefix_cache_retention_interval=prefix_cache_retention_interval,
|
||||
)
|
||||
|
||||
|
||||
@@ -1594,6 +1597,7 @@ def test_requester_worker_init_uses_positional_setup(tmp_path, monkeypatch):
|
||||
"mlx5_0",
|
||||
"10.0.0.7:50051",
|
||||
)
|
||||
assert w.coord.retention_interval == 0
|
||||
|
||||
|
||||
def test_requester_worker_init_prefers_local_hostname_override(
|
||||
|
||||
+22
-1
@@ -7,7 +7,7 @@ from typing import Any, ClassVar, Literal
|
||||
|
||||
from pydantic import Field, field_validator, model_validator
|
||||
|
||||
from vllm.config.utils import config
|
||||
from vllm.config.utils import config, get_from_deprecated_env_if_set
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.torch_utils import (
|
||||
is_quantized_kv_cache,
|
||||
@@ -35,6 +35,17 @@ CacheDType = Literal[
|
||||
"nvfp4",
|
||||
"nvfp4_4over6",
|
||||
]
|
||||
|
||||
|
||||
def _get_prefix_cache_retention_interval() -> int | None:
|
||||
env_value = get_from_deprecated_env_if_set(
|
||||
"VLLM_PREFIX_CACHE_RETENTION_INTERVAL",
|
||||
"v0.29",
|
||||
"prefix_cache_retention_interval",
|
||||
)
|
||||
return 0 if env_value is None else int(env_value)
|
||||
|
||||
|
||||
MambaDType = Literal["auto", "float32", "float16", "bfloat16"]
|
||||
MambaCacheMode = Literal["all", "align", "none"]
|
||||
PrefixCachingHashAlgo = Literal["sha256", "sha256_cbor", "xxhash", "xxhash_cbor"]
|
||||
@@ -111,6 +122,15 @@ class CacheConfig:
|
||||
security risk tolerance against the performance benefits before turning this on.
|
||||
- "xxhash_cbor" combines canonical CBOR serialization with xxHash for
|
||||
reproducible hashing. Requires the optional ``xxhash`` package."""
|
||||
prefix_cache_retention_interval: int | None = Field(
|
||||
default_factory=_get_prefix_cache_retention_interval, ge=0
|
||||
)
|
||||
"""Token interval between retained sliding-window and Mamba prefix-cache
|
||||
checkpoints. ``0`` retains only semantic checkpoints, including the latest
|
||||
replay boundary and shared-prefix junctions. Positive values additionally
|
||||
retain periodic checkpoints at the specified interval, which must be a
|
||||
multiple of the scheduler block size. ``None`` retains checkpoints densely.
|
||||
Applies only to sliding-window and Mamba cache groups."""
|
||||
kv_cache_dtype_skip_layers: list[str] = field(default_factory=list)
|
||||
"""Layer patterns to skip KV cache quantization. Accepts layer indices
|
||||
(e.g., '0', '2', '4') or attention type names (e.g., 'sliding_window')."""
|
||||
@@ -218,6 +238,7 @@ class CacheConfig:
|
||||
"num_gpu_blocks_override",
|
||||
"enable_prefix_caching",
|
||||
"prefix_caching_hash_algo",
|
||||
"prefix_cache_retention_interval",
|
||||
# Prefix-caching implementation detail (doesn't affect compiled graph).
|
||||
"prefix_match_unit",
|
||||
"mamba_page_size_padded",
|
||||
|
||||
@@ -1433,7 +1433,7 @@ class MooncakeStoreWorker:
|
||||
scheduler_block_size=self.block_size,
|
||||
hash_block_size=self.hash_block_size,
|
||||
use_eagle=use_eagle,
|
||||
retention_interval=envs.VLLM_PREFIX_CACHE_RETENTION_INTERVAL,
|
||||
retention_interval=kv_cache_config.prefix_cache_retention_interval,
|
||||
)
|
||||
# One ChunkedTokenDatabase per group; addresses populated in
|
||||
# register_kv_caches once the kv-cache layout is known. Each group's
|
||||
|
||||
@@ -522,6 +522,9 @@ class EngineArgs:
|
||||
prefix_caching_hash_algo: PrefixCachingHashAlgo = (
|
||||
CacheConfig.prefix_caching_hash_algo
|
||||
)
|
||||
prefix_cache_retention_interval: int | None = get_field(
|
||||
CacheConfig, "prefix_cache_retention_interval"
|
||||
)
|
||||
disable_sliding_window: bool = ModelConfig.disable_sliding_window
|
||||
disable_cascade_attn: bool = ModelConfig.disable_cascade_attn
|
||||
offload_backend: str = OffloadConfig.offload_backend
|
||||
@@ -1224,6 +1227,10 @@ class EngineArgs:
|
||||
cache_group.add_argument(
|
||||
"--prefix-caching-hash-algo", **cache_kwargs["prefix_caching_hash_algo"]
|
||||
)
|
||||
cache_group.add_argument(
|
||||
"--prefix-cache-retention-interval",
|
||||
**cache_kwargs["prefix_cache_retention_interval"],
|
||||
)
|
||||
cache_group.add_argument(
|
||||
"--kv-cache-dtype-skip-layers", **cache_kwargs["kv_cache_dtype_skip_layers"]
|
||||
)
|
||||
@@ -2001,6 +2008,7 @@ class EngineArgs:
|
||||
sliding_window=sliding_window,
|
||||
enable_prefix_caching=self.enable_prefix_caching,
|
||||
prefix_caching_hash_algo=self.prefix_caching_hash_algo,
|
||||
prefix_cache_retention_interval=self.prefix_cache_retention_interval,
|
||||
kv_cache_dtype_skip_layers=self.kv_cache_dtype_skip_layers,
|
||||
kv_sharing_fast_prefill=self.kv_sharing_fast_prefill,
|
||||
mamba_cache_dtype=self.mamba_cache_dtype,
|
||||
|
||||
@@ -1148,12 +1148,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
if "VLLM_PLUGINS" not in os.environ
|
||||
else os.environ["VLLM_PLUGINS"].split(",")
|
||||
),
|
||||
# Retain local sliding-window KV checkpoints for prefix caching.
|
||||
# Unset (default) preserves the dense local checkpointing behavior. `0`
|
||||
# retains only the latest completed prompt boundary. Positive values retain
|
||||
# checkpoints at the specified interval boundaries (rounded up to the
|
||||
# prefix-cache alignment).
|
||||
# Applies to sliding-window attention for now but not yet Mamba/linear attention.
|
||||
"VLLM_PREFIX_CACHE_RETENTION_INTERVAL": lambda: (
|
||||
int(os.environ["VLLM_PREFIX_CACHE_RETENTION_INTERVAL"])
|
||||
if "VLLM_PREFIX_CACHE_RETENTION_INTERVAL" in os.environ
|
||||
|
||||
@@ -4,7 +4,6 @@ from abc import ABC, abstractmethod
|
||||
from collections.abc import Sequence
|
||||
from typing import NamedTuple
|
||||
|
||||
from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv, round_down
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
@@ -45,16 +44,18 @@ def _validate_prefix_cache_retention_interval(
|
||||
isinstance(g.kv_cache_spec, (SlidingWindowSpec, MambaSpec))
|
||||
for g in kv_cache_config.kv_cache_groups
|
||||
):
|
||||
if retention_interval == 0:
|
||||
return
|
||||
raise ValueError(
|
||||
"VLLM_PREFIX_CACHE_RETENTION_INTERVAL is set but this model has "
|
||||
"prefix_cache_retention_interval is set but this model has "
|
||||
"no sliding-window or Mamba KV cache group, so retention has no "
|
||||
"effect. Unset it (it only applies to sliding-window and Mamba "
|
||||
"effect. Set it to 0 (it only applies to sliding-window and Mamba "
|
||||
"attention)."
|
||||
)
|
||||
|
||||
if retention_interval < 0 or retention_interval % scheduler_block_size != 0:
|
||||
raise ValueError(
|
||||
f"VLLM_PREFIX_CACHE_RETENTION_INTERVAL ({retention_interval}) "
|
||||
f"prefix_cache_retention_interval ({retention_interval}) "
|
||||
"must be non-negative and a multiple of scheduler_block_size "
|
||||
f"({scheduler_block_size})."
|
||||
)
|
||||
@@ -151,7 +152,7 @@ class KVCacheCoordinator(ABC):
|
||||
# A positive retention interval must be a multiple of the base hit granularity
|
||||
# (``scheduler_block_size``) to land on real cache-hit boundaries.
|
||||
# 0 = keep only the latest replay boundary; None = dense;
|
||||
self.retention_interval = envs.VLLM_PREFIX_CACHE_RETENTION_INTERVAL
|
||||
self.retention_interval = kv_cache_config.prefix_cache_retention_interval
|
||||
_validate_prefix_cache_retention_interval(
|
||||
self.retention_interval, self.scheduler_block_size, kv_cache_config
|
||||
)
|
||||
|
||||
@@ -243,7 +243,7 @@ class KVCacheManager:
|
||||
- ``shared_prefix_boundary``: the block-aligned token position of
|
||||
a shared prefix that a sparse-retention group (Mamba / sliding
|
||||
window) has not cached yet (Marconi-style APC), or 0 if none.
|
||||
Pinned so ``VLLM_PREFIX_CACHE_RETENTION_INTERVAL`` does not drop
|
||||
Pinned so sparse prefix-cache retention does not drop
|
||||
the junction and defeat cross-request reuse.
|
||||
"""
|
||||
# We skip finding the prefix cache hit when prefix caching is
|
||||
|
||||
@@ -1347,6 +1347,9 @@ def get_kv_cache_config_from_groups(
|
||||
num_blocks=1,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
prefix_cache_retention_interval=(
|
||||
vllm_config.cache_config.prefix_cache_retention_interval
|
||||
),
|
||||
)
|
||||
|
||||
# Determine how model runners should initialize the KV cache tensors.
|
||||
@@ -1406,6 +1409,9 @@ def get_kv_cache_config_from_groups(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=kv_cache_tensors,
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
prefix_cache_retention_interval=(
|
||||
vllm_config.cache_config.prefix_cache_retention_interval
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -970,6 +970,8 @@ class KVCacheConfig:
|
||||
For models with multiple types of attention, there will be multiple groups,
|
||||
see `_get_kv_cache_config_uniform_page_size` for more details.
|
||||
"""
|
||||
prefix_cache_retention_interval: int | None = None
|
||||
"""Resolved retention policy for local prefix-cache checkpoints."""
|
||||
|
||||
@property
|
||||
def has_mamba_layers(self) -> bool:
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
import contextlib
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass, field, replace
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
@@ -190,7 +190,6 @@ class SimpleCPUOffloadScheduler:
|
||||
"""Derive a CPU KVCacheConfig from the GPU config.
|
||||
Same kv_cache_groups, num_blocks scaled by CPU/GPU memory ratio."""
|
||||
# Import here to avoid potential circular imports
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig as KVCacheConfigCls
|
||||
from vllm.v1.kv_cache_interface import KVCacheTensor
|
||||
|
||||
assert len(gpu_config.kv_cache_tensors) > 0
|
||||
@@ -215,10 +214,10 @@ class SimpleCPUOffloadScheduler:
|
||||
for t in gpu_config.kv_cache_tensors
|
||||
]
|
||||
|
||||
return KVCacheConfigCls(
|
||||
return replace(
|
||||
gpu_config,
|
||||
num_blocks=num_cpu_blocks,
|
||||
kv_cache_tensors=cpu_tensors,
|
||||
kv_cache_groups=gpu_config.kv_cache_groups,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
||||
Reference in New Issue
Block a user