[Core] Add kvcache watermark to reduce preemptions (#44594)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
23eb7c8fbb
commit
4085ff7cb4
@@ -1849,6 +1849,8 @@ def create_scheduler_with_priority(
|
||||
enable_chunked_prefill=True,
|
||||
is_encoder_decoder=model_config.is_encoder_decoder,
|
||||
policy="priority", # Enable priority scheduling
|
||||
# Ensure admission/preemption mechanics are deterministic
|
||||
watermark=0.0,
|
||||
)
|
||||
# Cache config, optionally force APC
|
||||
cache_config = CacheConfig(
|
||||
|
||||
@@ -90,6 +90,8 @@ def create_scheduler(
|
||||
enable_chunked_prefill=enable_chunked_prefill,
|
||||
async_scheduling=async_scheduling,
|
||||
is_encoder_decoder=model_config.is_encoder_decoder,
|
||||
# Ensure admission/preemption mechanics are deterministic
|
||||
watermark=0.0,
|
||||
)
|
||||
# Cache config, optionally force APC
|
||||
cache_config = CacheConfig(
|
||||
|
||||
Reference in New Issue
Block a user