Add --max-model-len auto to auto-fit context to available memory (#29431)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -247,9 +247,20 @@ class EngineCore:
|
||||
|
||||
assert len(kv_cache_specs) == len(available_gpu_memory)
|
||||
|
||||
# Track max_model_len before KV cache config to detect auto-fit changes
|
||||
max_model_len_before = vllm_config.model_config.max_model_len
|
||||
|
||||
kv_cache_configs = get_kv_cache_configs(
|
||||
vllm_config, kv_cache_specs, available_gpu_memory
|
||||
)
|
||||
|
||||
# If auto-fit reduced max_model_len, sync the new value to workers.
|
||||
# This is needed because workers were spawned before memory profiling
|
||||
# and have the original (larger) max_model_len cached.
|
||||
max_model_len_after = vllm_config.model_config.max_model_len
|
||||
if max_model_len_after != max_model_len_before:
|
||||
self.collective_rpc("update_max_model_len", args=(max_model_len_after,))
|
||||
|
||||
scheduler_kv_cache_config = generate_scheduler_kv_cache_config(kv_cache_configs)
|
||||
num_gpu_blocks = scheduler_kv_cache_config.num_blocks
|
||||
num_cpu_blocks = 0
|
||||
|
||||
Reference in New Issue
Block a user