[Core] Add kvcache watermark to reduce preemptions (#44594)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Nick Hill
2026-06-11 08:27:31 -07:00
committed by GitHub
co-authored by Claude Opus 4.8
parent 23eb7c8fbb
commit 4085ff7cb4
7 changed files with 291 additions and 8 deletions
+2
View File
@@ -1849,6 +1849,8 @@ def create_scheduler_with_priority(
enable_chunked_prefill=True,
is_encoder_decoder=model_config.is_encoder_decoder,
policy="priority", # Enable priority scheduling
# Ensure admission/preemption mechanics are deterministic
watermark=0.0,
)
# Cache config, optionally force APC
cache_config = CacheConfig(
+2
View File
@@ -90,6 +90,8 @@ def create_scheduler(
enable_chunked_prefill=enable_chunked_prefill,
async_scheduling=async_scheduling,
is_encoder_decoder=model_config.is_encoder_decoder,
# Ensure admission/preemption mechanics are deterministic
watermark=0.0,
)
# Cache config, optionally force APC
cache_config = CacheConfig(