Promote prefix_cache_retention_interval to an argument and change the default to 0 (#52216)
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
1d3a8b9e22
commit
017e9f4448
@@ -216,6 +216,11 @@ def test_cache_config_hash_ignores_kv_cache_sizing_knobs():
|
||||
assert CacheConfig(gpu_memory_utilization=0.5).compute_hash() == base_hash
|
||||
|
||||
|
||||
def test_cache_config_hash_ignores_prefix_cache_retention_interval():
|
||||
base_hash = CacheConfig().compute_hash()
|
||||
assert CacheConfig(prefix_cache_retention_interval=64).compute_hash() == base_hash
|
||||
|
||||
|
||||
def test_envs_compile_factors_relocation_invariant(tmp_path):
|
||||
"""Relocating HOME or the XDG roots must not change the compile-cache
|
||||
env hash.
|
||||
|
||||
@@ -476,6 +476,7 @@ def test_prefix_cache_default():
|
||||
# should be None by default (depends on model).
|
||||
engine_args = EngineArgs.from_cli_args(args=args)
|
||||
assert engine_args.enable_prefix_caching is None
|
||||
assert engine_args.prefix_cache_retention_interval == 0
|
||||
|
||||
# with flag to turn it on.
|
||||
args = parser.parse_args(["--enable-prefix-caching"])
|
||||
@@ -487,6 +488,28 @@ def test_prefix_cache_default():
|
||||
engine_args = EngineArgs.from_cli_args(args=args)
|
||||
assert not engine_args.enable_prefix_caching
|
||||
|
||||
args = parser.parse_args(["--prefix-cache-retention-interval", "64"])
|
||||
engine_args = EngineArgs.from_cli_args(args=args)
|
||||
assert engine_args.prefix_cache_retention_interval == 64
|
||||
|
||||
|
||||
def test_prefix_cache_retention_interval_from_deprecated_env(
|
||||
monkeypatch, caplog, disable_log_dedup
|
||||
):
|
||||
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "64")
|
||||
|
||||
engine_args = EngineArgs()
|
||||
|
||||
assert engine_args.prefix_cache_retention_interval == 64
|
||||
assert "VLLM_PREFIX_CACHE_RETENTION_INTERVAL" in caplog.text
|
||||
assert "deprecated" in caplog.text
|
||||
assert "prefix_cache_retention_interval" in caplog.text
|
||||
|
||||
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
|
||||
args = parser.parse_args(["--prefix-cache-retention-interval", "32"])
|
||||
engine_args = EngineArgs.from_cli_args(args)
|
||||
assert engine_args.prefix_cache_retention_interval == 32
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("arg", "expected", "option"),
|
||||
|
||||
@@ -88,6 +88,7 @@ def _make_groups(n_c4, n_c128, n_swa):
|
||||
def _mock_vllm_config(kv_connector_extra_config: dict[str, str] | None = None):
|
||||
config = MagicMock()
|
||||
config.cache_config.num_gpu_blocks_override = None
|
||||
config.cache_config.prefix_cache_retention_interval = 0
|
||||
config.kv_transfer_config = None
|
||||
if kv_connector_extra_config is not None:
|
||||
config.kv_transfer_config = MagicMock()
|
||||
@@ -322,6 +323,7 @@ class TestInterleavedPacking:
|
||||
)
|
||||
|
||||
assert config.num_blocks == 32
|
||||
assert config.prefix_cache_retention_interval == 0
|
||||
assert sum(t.size for t in config.kv_cache_tensors) == page_size * 2 * 32
|
||||
assert config.kv_cache_tensors == [
|
||||
KVCacheTensor(size=page_size * 32, shared_by=["full.0", "sw.0", "sw.1"]),
|
||||
|
||||
@@ -816,6 +816,7 @@ def test_metrics_empty_stats():
|
||||
def test_get_kv_cache_configs_multiple_workers():
|
||||
model_config = ModelConfig(max_model_len=16)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
vllm_config.cache_config.prefix_cache_retention_interval = None
|
||||
|
||||
ref_kv_cache_spec = new_kv_cache_spec()
|
||||
same_kv_cache_specs = [
|
||||
@@ -1173,6 +1174,7 @@ def test_get_kv_cache_configs_multiple_workers():
|
||||
def test_get_kv_cache_configs_pp_sharding(asymmetric_memory):
|
||||
model_config = ModelConfig(max_model_len=512)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
vllm_config.cache_config.prefix_cache_retention_interval = None
|
||||
|
||||
ref_kv_cache_spec = new_kv_cache_spec()
|
||||
pp_kv_cache_specs = [
|
||||
@@ -1702,6 +1704,7 @@ def test_get_kv_cache_config_one_worker():
|
||||
# pass max_model_len to pass check_enough_kv_cache_memory
|
||||
model_config = ModelConfig(max_model_len=16)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
vllm_config.cache_config.prefix_cache_retention_interval = None
|
||||
|
||||
mem_per_block_per_layer = 16 * 2 * 64 * 4 * 2
|
||||
# all layers are full attention -> single group
|
||||
@@ -2015,6 +2018,7 @@ def test_get_kv_cache_config_one_worker():
|
||||
def test_get_kv_cache_configs_attention_free():
|
||||
kv_cache_specs: dict[str, KVCacheSpec] = {}
|
||||
vllm_config = VllmConfig(model_config=ModelConfig(max_model_len=16))
|
||||
vllm_config.cache_config.prefix_cache_retention_interval = None
|
||||
kv_cache_configs = get_kv_cache_configs(vllm_config, [kv_cache_specs], [0])
|
||||
assert kv_cache_configs == [
|
||||
KVCacheConfig(
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
import copy
|
||||
from collections.abc import Callable
|
||||
from dataclasses import replace
|
||||
from math import lcm
|
||||
from types import SimpleNamespace
|
||||
|
||||
@@ -110,6 +111,11 @@ def make_kv_cache_manager(kv_cache_config: KVCacheConfig, **kwargs) -> KVCacheMa
|
||||
"scheduler_block_size",
|
||||
lcm(*(g.kv_cache_spec.block_size for g in kv_cache_config.kv_cache_groups)),
|
||||
)
|
||||
if "retention_interval" in kwargs:
|
||||
kv_cache_config = replace(
|
||||
kv_cache_config,
|
||||
prefix_cache_retention_interval=kwargs.pop("retention_interval"),
|
||||
)
|
||||
return KVCacheManager(kv_cache_config, **kwargs)
|
||||
|
||||
|
||||
@@ -3119,9 +3125,8 @@ def test_hybrid_cache_blocks_clamped_to_lcm():
|
||||
)
|
||||
|
||||
|
||||
def test_hybrid_local_kv_retention_interval_aligns_in_manager(monkeypatch):
|
||||
def test_hybrid_local_kv_retention_interval_aligns_in_manager():
|
||||
"""Verify fixed intervals retain sparse tails plus the latest replay tail."""
|
||||
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "64")
|
||||
block_size = 8
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=100,
|
||||
@@ -3153,6 +3158,7 @@ def test_hybrid_local_kv_retention_interval_aligns_in_manager(monkeypatch):
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
retention_interval=64,
|
||||
)
|
||||
|
||||
# The SWA manager uses the configured 64-token interval (a multiple of the
|
||||
@@ -3184,18 +3190,15 @@ def test_hybrid_local_kv_retention_interval_aligns_in_manager(monkeypatch):
|
||||
"interval, expected_match",
|
||||
[
|
||||
# scheduler_block_size is 32 (= lcm(4*8, 8)); 33 is not a multiple of it.
|
||||
("33", "multiple of scheduler_block_size"),
|
||||
(33, "multiple of scheduler_block_size"),
|
||||
# A negative multiple (-32 % 32 == 0) must still be rejected explicitly,
|
||||
# otherwise it would pass the modulo check and silently degrade to dense.
|
||||
("-32", "non-negative"),
|
||||
(-32, "non-negative"),
|
||||
],
|
||||
)
|
||||
def test_hybrid_local_kv_retention_interval_rejects_invalid(
|
||||
monkeypatch, interval, expected_match
|
||||
):
|
||||
def test_hybrid_local_kv_retention_interval_rejects_invalid(interval, expected_match):
|
||||
"""A retention interval that is negative or not a multiple of
|
||||
scheduler_block_size errors out at construction time."""
|
||||
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", interval)
|
||||
block_size = 8
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=100,
|
||||
@@ -3228,12 +3231,36 @@ def test_hybrid_local_kv_retention_interval_rejects_invalid(
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
retention_interval=interval,
|
||||
)
|
||||
|
||||
|
||||
def test_hybrid_local_kv_retention_interval_survives_recycling(monkeypatch):
|
||||
def test_zero_retention_is_ignored_for_full_attention():
|
||||
kv_cache_config = make_kv_cache_config(block_size=16, num_blocks=10)
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=16,
|
||||
retention_interval=0,
|
||||
)
|
||||
assert manager.coordinator.retention_interval == 0
|
||||
|
||||
|
||||
def test_positive_retention_rejects_full_attention():
|
||||
kv_cache_config = make_kv_cache_config(block_size=16, num_blocks=10)
|
||||
with pytest.raises(ValueError, match="no sliding-window or Mamba"):
|
||||
make_kv_cache_manager(
|
||||
kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=16,
|
||||
retention_interval=16,
|
||||
)
|
||||
|
||||
|
||||
def test_hybrid_local_kv_retention_interval_survives_recycling():
|
||||
"""Verify retained local checkpoints are reused after block recycling."""
|
||||
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "1024")
|
||||
hash_block_size = 4
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=800,
|
||||
@@ -3286,6 +3313,7 @@ def test_hybrid_local_kv_retention_interval_survives_recycling(monkeypatch):
|
||||
max_model_len=4096,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
retention_interval=1024,
|
||||
)
|
||||
|
||||
def fill_request(request_id: str, token_offset: int) -> list[int]:
|
||||
@@ -3314,9 +3342,8 @@ def test_hybrid_local_kv_retention_interval_survives_recycling(monkeypatch):
|
||||
assert [len(blocks) for blocks in computed_blocks.blocks] == [4, 16, 128, 256]
|
||||
|
||||
|
||||
def test_hybrid_local_kv_retention_latest_only_reuses_replay_boundary(monkeypatch):
|
||||
def test_hybrid_local_kv_retention_latest_only_reuses_replay_boundary():
|
||||
"""Verify latest-only retention reuses only the replayable prompt boundary."""
|
||||
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "0")
|
||||
block_size = 8
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=100,
|
||||
@@ -3348,6 +3375,7 @@ def test_hybrid_local_kv_retention_latest_only_reuses_replay_boundary(monkeypatc
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
retention_interval=0,
|
||||
)
|
||||
|
||||
token_ids = [i for i in range(16) for _ in range(block_size)]
|
||||
@@ -3388,14 +3416,13 @@ def test_hybrid_local_kv_retention_latest_only_reuses_replay_boundary(monkeypatc
|
||||
assert len(computed_blocks.blocks[1]) == 0
|
||||
|
||||
|
||||
def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary(monkeypatch):
|
||||
def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary():
|
||||
"""Verify MTP/EAGLE SWA retention keeps the extra proof block.
|
||||
|
||||
EAGLE/MTP lookup matches one additional local block after the returned
|
||||
prefix and then drops it. Sparse retention must therefore cache the normal
|
||||
local tail at the latest replay boundary plus one extra SWA block.
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "0")
|
||||
block_size = 8
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=100,
|
||||
@@ -3428,6 +3455,7 @@ def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary(monkeypatch):
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
retention_interval=0,
|
||||
use_eagle=True,
|
||||
)
|
||||
|
||||
@@ -3824,12 +3852,11 @@ def test_cache_hit_local_and_external_two_groups_preempt_and_reallocate():
|
||||
assert manager.get_blocks("test").get_block_ids() != ([], [])
|
||||
|
||||
|
||||
def test_swa_free_split_keeps_cached_tail_ahead_of_scratch(monkeypatch):
|
||||
"""Default path (no retention): freeing an SWA request must place its
|
||||
def test_swa_free_split_keeps_cached_tail_ahead_of_scratch():
|
||||
"""Dense retention: freeing an SWA request must place its
|
||||
uncached scratch blocks at the front of the free queue (recycled first)
|
||||
and keep its cached checkpoint blocks at the back (retained for prefix
|
||||
hits). This split is always-on, independent of the retention interval."""
|
||||
monkeypatch.delenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", raising=False)
|
||||
block_size = 8
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=100,
|
||||
@@ -3937,13 +3964,14 @@ def _make_pure_swa_manager(block_size, sliding_window, num_blocks=100, **kwargs)
|
||||
)
|
||||
|
||||
|
||||
def test_pure_swa_retention_interval_caches_sparse_tails(monkeypatch):
|
||||
def test_pure_swa_retention_interval_caches_sparse_tails():
|
||||
"""Sparse retention must work for a pure-SWA single-group model, not just
|
||||
hybrid models: only the per-interval tails plus the latest replay tail are
|
||||
cached, and a replay still hits the latest replayable boundary."""
|
||||
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "64")
|
||||
block_size = 16
|
||||
manager = _make_pure_swa_manager(block_size, sliding_window=block_size)
|
||||
manager = _make_pure_swa_manager(
|
||||
block_size, sliding_window=block_size, retention_interval=64
|
||||
)
|
||||
assert type(manager.coordinator).__name__ == "UnitaryKVCacheCoordinator"
|
||||
|
||||
token_ids = [i for i in range(16) for _ in range(block_size)]
|
||||
@@ -3976,11 +4004,12 @@ def test_pure_swa_retention_interval_caches_sparse_tails(monkeypatch):
|
||||
assert num_computed == 240
|
||||
|
||||
|
||||
def test_pure_swa_retention_latest_only(monkeypatch):
|
||||
def test_pure_swa_retention_latest_only():
|
||||
"""`=0` on a pure-SWA model keeps only the latest replay tail."""
|
||||
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "0")
|
||||
block_size = 16
|
||||
manager = _make_pure_swa_manager(block_size, sliding_window=block_size)
|
||||
manager = _make_pure_swa_manager(
|
||||
block_size, sliding_window=block_size, retention_interval=0
|
||||
)
|
||||
|
||||
token_ids = [i for i in range(16) for _ in range(block_size)]
|
||||
req = make_request("0", token_ids, block_size, sha256)
|
||||
@@ -4008,10 +4037,9 @@ def test_pure_swa_retention_latest_only(monkeypatch):
|
||||
assert num_computed == 240
|
||||
|
||||
|
||||
def test_pure_swa_retention_dense_default_caches_all(monkeypatch):
|
||||
"""With retention unset, a pure-SWA model must keep the dense behavior:
|
||||
def test_pure_swa_dense_retention_caches_all():
|
||||
"""With retention set to ``None``, a pure-SWA model keeps dense behavior:
|
||||
every block boundary is a potential hit, so all blocks are cached."""
|
||||
monkeypatch.delenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", raising=False)
|
||||
block_size = 16
|
||||
manager = _make_pure_swa_manager(block_size, sliding_window=block_size)
|
||||
|
||||
@@ -4037,7 +4065,7 @@ def test_pure_swa_retention_dense_default_caches_all(monkeypatch):
|
||||
|
||||
|
||||
def test_mamba_reachable_block_mask_sparsifies_retention():
|
||||
"""Mamba state-snapshot retention: with VLLM_PREFIX_CACHE_RETENTION_INTERVAL
|
||||
"""Mamba state-snapshot retention: with a configured retention interval,
|
||||
the manager keeps one cached state per interval-sized segment (plus the
|
||||
latest replay boundary) instead of a snapshot per block, which is what
|
||||
lets a small attention block_size avoid Mamba dominating the KV pool."""
|
||||
@@ -4063,7 +4091,7 @@ def test_mamba_reachable_block_mask_sparsifies_retention():
|
||||
)
|
||||
return None if m is None else {i for i, v in enumerate(m) if v}
|
||||
|
||||
# Dense default (None) -> no mask, every block cached (unchanged behavior).
|
||||
# Dense retention (None) -> no mask, every block cached.
|
||||
assert retained(None) is None
|
||||
# interval == block_size -> every block is a boundary -> stays dense.
|
||||
assert retained(block_size) is None
|
||||
@@ -4111,7 +4139,7 @@ def test_mamba_reachable_block_mask_pins_shared_prefix():
|
||||
assert retained(0, 100) == {5, 14}
|
||||
# Coexists with segment tails (interval 64 -> {3,7,11,15} + replay 14).
|
||||
assert retained(64, 96) == {3, 5, 7, 11, 14, 15}
|
||||
# Dense default ignores the hint (nothing to sparsify).
|
||||
# Dense retention ignores the hint (nothing to sparsify).
|
||||
assert retained(None, 96) is None
|
||||
# Out-of-range boundary is a no-op (only replay 14 remains).
|
||||
assert retained(0, 16 * block_size * 2) == {14}
|
||||
@@ -4120,14 +4148,13 @@ def test_mamba_reachable_block_mask_pins_shared_prefix():
|
||||
assert retained(0, None) == {14}
|
||||
|
||||
|
||||
def test_mamba_shared_prefix_survives_zero_retention(monkeypatch):
|
||||
def test_mamba_shared_prefix_survives_zero_retention():
|
||||
"""Manager-level check of the full wiring: a pinned shared-prefix boundary
|
||||
(``Request.shared_prefix_boundary``, set by the scheduler on Marconi-style
|
||||
detection) keeps its Mamba state block cached under
|
||||
``VLLM_PREFIX_CACHE_RETENTION_INTERVAL=0``, which otherwise retains only the
|
||||
``prefix_cache_retention_interval=0``, which otherwise retains only the
|
||||
end-of-prompt replay boundary. Without this, a shared prefix (junction
|
||||
before ``num_prompt``) would be recomputed by every sharing request."""
|
||||
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "0")
|
||||
block_size = 16
|
||||
|
||||
# 16-block (256-token) prompt; replay boundary is block 240 // 16 - 1 = 14.
|
||||
@@ -4140,6 +4167,7 @@ def test_mamba_shared_prefix_survives_zero_retention(monkeypatch):
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
retention_interval=0,
|
||||
)
|
||||
req = make_request("r", token_ids, block_size, sha256)
|
||||
req.shared_prefix_boundary = shared_prefix_boundary
|
||||
@@ -4163,24 +4191,21 @@ def test_mamba_shared_prefix_survives_zero_retention(monkeypatch):
|
||||
assert cached_mamba_blocks(96) == {5, 14}
|
||||
|
||||
|
||||
def test_mamba_shared_prefix_reuse_under_zero_retention(monkeypatch):
|
||||
def test_mamba_shared_prefix_reuse_under_zero_retention():
|
||||
"""Full cross-request Marconi flow: a partial shared prefix cached by the
|
||||
detecting request must stay reusable by a later request under
|
||||
``VLLM_PREFIX_CACHE_RETENTION_INTERVAL=0``. Without the pin the junction is
|
||||
``prefix_cache_retention_interval=0``. Without the pin the junction is
|
||||
masked out and the later request misses; with it (and under dense) the reuse
|
||||
is preserved."""
|
||||
block_size = 16
|
||||
|
||||
def last_req_hit(retention, pin):
|
||||
if retention is None:
|
||||
monkeypatch.delenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", raising=False)
|
||||
else:
|
||||
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", str(retention))
|
||||
manager = make_kv_cache_manager(
|
||||
_make_hybrid_kv_cache_config(block_size, 200, ["full", "mamba_align"]),
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
retention_interval=retention,
|
||||
)
|
||||
shared = [7 for _ in range(2 * block_size)] # 2-block shared prefix
|
||||
|
||||
@@ -4261,23 +4286,20 @@ def test_swa_reachable_block_mask_pins_shared_prefix():
|
||||
assert retained(0, 0, block_size) == {14}
|
||||
|
||||
|
||||
def test_swa_shared_prefix_reuse_under_zero_retention(monkeypatch):
|
||||
def test_swa_shared_prefix_reuse_under_zero_retention():
|
||||
"""SWA cross-request analog: a partial shared prefix's sliding-window tail
|
||||
must stay reusable under ``VLLM_PREFIX_CACHE_RETENTION_INTERVAL=0``. Without
|
||||
must stay reusable under ``prefix_cache_retention_interval=0``. Without
|
||||
the pin the junction window is masked out and a later request misses; with
|
||||
it (and under dense) reuse is preserved."""
|
||||
block_size = 16
|
||||
|
||||
def last_req_hit(retention, pin):
|
||||
if retention is None:
|
||||
monkeypatch.delenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", raising=False)
|
||||
else:
|
||||
monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", str(retention))
|
||||
manager = make_kv_cache_manager(
|
||||
_make_hybrid_kv_cache_config(block_size, 200, ["full", "sliding_window"]),
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
retention_interval=retention,
|
||||
)
|
||||
shared = [7 for _ in range(4 * block_size)] # 4-block shared prefix
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ def test_prefix_caching_from_cli():
|
||||
assert vllm_config.cache_config.enable_prefix_caching, (
|
||||
"V1 turns on prefix caching by default."
|
||||
)
|
||||
assert vllm_config.cache_config.prefix_cache_retention_interval == 0
|
||||
|
||||
# Turn it off possible with flag.
|
||||
args = parser.parse_args(["--no-enable-prefix-caching"])
|
||||
@@ -47,6 +48,10 @@ def test_prefix_caching_from_cli():
|
||||
with pytest.raises(ArgumentError):
|
||||
args = parser.parse_args(["--prefix-caching-hash-algo", "invalid"])
|
||||
|
||||
args = parser.parse_args(["--prefix-cache-retention-interval", "64"])
|
||||
vllm_config = EngineArgs.from_cli_args(args=args).create_engine_config()
|
||||
assert vllm_config.cache_config.prefix_cache_retention_interval == 64
|
||||
|
||||
|
||||
@pytest.mark.skipif(_xxhash is None, reason="xxhash not installed")
|
||||
def test_prefix_caching_xxhash_from_cli():
|
||||
|
||||
@@ -1066,6 +1066,7 @@ def test_kv_cache_events(
|
||||
model=model_name,
|
||||
enforce_eager=True,
|
||||
enable_prefix_caching=True,
|
||||
prefix_cache_retention_interval=None,
|
||||
block_size=block_size,
|
||||
)
|
||||
engine_args.kv_events_config = publisher_config
|
||||
|
||||
@@ -247,7 +247,9 @@ def _make_vllm_config(
|
||||
)
|
||||
|
||||
|
||||
def _make_kv_cache_config(*, block_size: int = 16) -> object:
|
||||
def _make_kv_cache_config(
|
||||
*, block_size: int = 16, prefix_cache_retention_interval: int | None = 0
|
||||
) -> object:
|
||||
"""Minimal single-group KVCacheConfig for topology tests."""
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
@@ -262,6 +264,7 @@ def _make_kv_cache_config(*, block_size: int = 16) -> object:
|
||||
num_blocks=10,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["layer0"], spec)],
|
||||
prefix_cache_retention_interval=prefix_cache_retention_interval,
|
||||
)
|
||||
|
||||
|
||||
@@ -1594,6 +1597,7 @@ def test_requester_worker_init_uses_positional_setup(tmp_path, monkeypatch):
|
||||
"mlx5_0",
|
||||
"10.0.0.7:50051",
|
||||
)
|
||||
assert w.coord.retention_interval == 0
|
||||
|
||||
|
||||
def test_requester_worker_init_prefers_local_hostname_override(
|
||||
|
||||
Reference in New Issue
Block a user