[Core] Add kvcache watermark to reduce preemptions (#44594)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Nick Hill
2026-06-11 08:27:31 -07:00
committed by GitHub
co-authored by Claude Opus 4.8
parent 23eb7c8fbb
commit 4085ff7cb4
7 changed files with 291 additions and 8 deletions
+4
View File
@@ -600,6 +600,8 @@ class EngineArgs:
scheduler_reserve_full_isl: bool = SchedulerConfig.scheduler_reserve_full_isl
watermark: float = SchedulerConfig.watermark
disable_hybrid_kv_cache_manager: bool | None = (
SchedulerConfig.disable_hybrid_kv_cache_manager
)
@@ -1408,6 +1410,7 @@ class EngineArgs:
"--scheduler-reserve-full-isl",
**scheduler_kwargs["scheduler_reserve_full_isl"],
)
scheduler_group.add_argument("--watermark", **scheduler_kwargs["watermark"])
scheduler_group.add_argument(
"--disable-hybrid-kv-cache-manager",
**scheduler_kwargs["disable_hybrid_kv_cache_manager"],
@@ -2045,6 +2048,7 @@ class EngineArgs:
max_long_partial_prefills=self.max_long_partial_prefills,
long_prefill_token_threshold=self.long_prefill_token_threshold,
scheduler_reserve_full_isl=self.scheduler_reserve_full_isl,
watermark=self.watermark,
disable_hybrid_kv_cache_manager=self.disable_hybrid_kv_cache_manager,
async_scheduling=self.async_scheduling,
stream_interval=self.stream_interval,