Promote prefix_cache_retention_interval to an argument and change the default to 0 (#52216)
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
1d3a8b9e22
commit
017e9f4448
@@ -970,6 +970,8 @@ class KVCacheConfig:
|
||||
For models with multiple types of attention, there will be multiple groups,
|
||||
see `_get_kv_cache_config_uniform_page_size` for more details.
|
||||
"""
|
||||
prefix_cache_retention_interval: int | None = None
|
||||
"""Resolved retention policy for local prefix-cache checkpoints."""
|
||||
|
||||
@property
|
||||
def has_mamba_layers(self) -> bool:
|
||||
|
||||
Reference in New Issue
Block a user