[Prefix Caching] DeepSeekv4 - Support selective prefix-cache retention for sliding-window KV cache (#43447)

Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Wei Zhao
2026-06-04 00:48:31 -07:00
committed by GitHub
co-authored by Yifan Qiao
parent 22c2e87555
commit a6183563b6
7 changed files with 792 additions and 45 deletions
+12
View File
@@ -279,6 +279,7 @@ if TYPE_CHECKING:
VLLM_LORA_ENABLE_DUAL_STREAM: bool = False
VLLM_GPU_NIC_PCIE_MAPPING: str = ""
VLLM_NIC_SELECTION_VARS: str = ""
VLLM_PREFIX_CACHE_RETENTION_INTERVAL: int | None = None
def get_default_cache_root():
@@ -1032,6 +1033,17 @@ environment_variables: dict[str, Callable[[], Any]] = {
if "VLLM_PLUGINS" not in os.environ
else os.environ["VLLM_PLUGINS"].split(",")
),
# Retain local sliding-window KV checkpoints for prefix caching.
# Unset (default) preserves the dense local checkpointing behavior. `0`
# retains only the latest completed prompt boundary. Positive values retain
# checkpoints at the specified interval boundaries (rounded up to the
# prefix-cache alignment).
# Applies to sliding-window attention for now but not yet Mamba/linear attention.
"VLLM_PREFIX_CACHE_RETENTION_INTERVAL": lambda: (
int(os.environ["VLLM_PREFIX_CACHE_RETENTION_INTERVAL"])
if "VLLM_PREFIX_CACHE_RETENTION_INTERVAL" in os.environ
else None
),
# a local directory to look in for unrecognized LoRA adapters.
# only works if plugins are enabled and
# VLLM_ALLOW_RUNTIME_LORA_UPDATING is enabled.