Promote prefix_cache_retention_interval to an argument and change the default to 0 (#52216)
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
1d3a8b9e22
commit
017e9f4448
@@ -522,6 +522,9 @@ class EngineArgs:
|
||||
prefix_caching_hash_algo: PrefixCachingHashAlgo = (
|
||||
CacheConfig.prefix_caching_hash_algo
|
||||
)
|
||||
prefix_cache_retention_interval: int | None = get_field(
|
||||
CacheConfig, "prefix_cache_retention_interval"
|
||||
)
|
||||
disable_sliding_window: bool = ModelConfig.disable_sliding_window
|
||||
disable_cascade_attn: bool = ModelConfig.disable_cascade_attn
|
||||
offload_backend: str = OffloadConfig.offload_backend
|
||||
@@ -1224,6 +1227,10 @@ class EngineArgs:
|
||||
cache_group.add_argument(
|
||||
"--prefix-caching-hash-algo", **cache_kwargs["prefix_caching_hash_algo"]
|
||||
)
|
||||
cache_group.add_argument(
|
||||
"--prefix-cache-retention-interval",
|
||||
**cache_kwargs["prefix_cache_retention_interval"],
|
||||
)
|
||||
cache_group.add_argument(
|
||||
"--kv-cache-dtype-skip-layers", **cache_kwargs["kv_cache_dtype_skip_layers"]
|
||||
)
|
||||
@@ -2001,6 +2008,7 @@ class EngineArgs:
|
||||
sliding_window=sliding_window,
|
||||
enable_prefix_caching=self.enable_prefix_caching,
|
||||
prefix_caching_hash_algo=self.prefix_caching_hash_algo,
|
||||
prefix_cache_retention_interval=self.prefix_cache_retention_interval,
|
||||
kv_cache_dtype_skip_layers=self.kv_cache_dtype_skip_layers,
|
||||
kv_sharing_fast_prefill=self.kv_sharing_fast_prefill,
|
||||
mamba_cache_dtype=self.mamba_cache_dtype,
|
||||
|
||||
Reference in New Issue
Block a user