From 5f2ee2fa8c59d9f9cddd07c79806e6418b0d841a Mon Sep 17 00:00:00 2001 From: Tanmay Dixit Date: Tue, 4 Aug 2026 08:53:32 -0400 Subject: [PATCH] [Bugfix][Core] Log KV cache capacity after block-size resolution (#50462) Signed-off-by: Tanmay Dixit <22117787+tandixit95@users.noreply.github.com> Signed-off-by: Michael Goin Signed-off-by: Matthew Bonanni Co-authored-by: Tanmay Dixit <22117787+tandixit95@users.noreply.github.com> Co-authored-by: Michael Goin Co-authored-by: Matthew Bonanni Co-authored-by: Claude Opus 5 (1M context) --- vllm/v1/core/kv_cache_utils.py | 34 +++++++++++++++++----------------- vllm/v1/engine/core.py | 8 ++------ 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/vllm/v1/core/kv_cache_utils.py b/vllm/v1/core/kv_cache_utils.py index 6c7f930d85..fab52a355d 100644 --- a/vllm/v1/core/kv_cache_utils.py +++ b/vllm/v1/core/kv_cache_utils.py @@ -1853,6 +1853,23 @@ def get_kv_cache_capacity( return int(max_concurrency * max_model_len), max_concurrency +def update_kv_cache_capacity( + vllm_config: VllmConfig, kv_cache_config: KVCacheConfig +) -> None: + """Store and log the resolved KV cache capacity.""" + num_tokens, max_concurrency = get_kv_cache_capacity(vllm_config, kv_cache_config) + vllm_config.cache_config.kv_cache_size_tokens = num_tokens + vllm_config.cache_config.kv_cache_max_concurrency = max_concurrency + max_model_len = vllm_config.model_config.max_model_len + logger.info_once( + "GPU KV cache size: %s tokens, " + "Maximum concurrency for %s tokens per request: %.2fx", + f"{num_tokens:,}", + f"{max_model_len:,}", + max_concurrency, + ) + + def _max_memory_usage_bytes_from_groups( vllm_config: VllmConfig, kv_cache_groups: list[KVCacheGroupSpec], @@ -2188,23 +2205,6 @@ def get_kv_cache_configs( assert tensor.size % num_blocks_old == 0 tensor.size = tensor.size // num_blocks_old * min_num_blocks - if len(kv_cache_config.kv_cache_groups) > 0: - max_model_len = vllm_config.model_config.max_model_len - # GPU KV cache size in tokens = max_concurrency * max_model_len: - # the total tokens of context the pool can hold at peak - # utilization. Sourcing this from the concurrency calculation - # handles hybrid layouts correctly. - num_tokens, max_concurrency = get_kv_cache_capacity( - vllm_config, kv_cache_config - ) - - logger.info_once("GPU KV cache size: %s tokens", f"{num_tokens:,}") - logger.info_once( - "Maximum concurrency for %s tokens per request: %.2fx", - f"{max_model_len:,}", - max_concurrency, - ) - return kv_cache_configs diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index 11c78c7e74..94d426d712 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -46,11 +46,11 @@ from vllm.utils.system_utils import decorate_logs, set_process_title from vllm.v1.core.kv_cache_utils import ( BlockHash, generate_scheduler_kv_cache_config, - get_kv_cache_capacity, get_kv_cache_configs, get_request_block_hasher, init_none_hash, resolve_kv_cache_block_sizes, + update_kv_cache_capacity, ) from vllm.v1.core.sched.interface import PauseState, SchedulerInterface from vllm.v1.core.sched.output import SchedulerOutput @@ -319,11 +319,7 @@ class EngineCore: vllm_config.cache_config.block_size = min( g.kv_cache_spec.block_size for g in kv_cache_groups ) - num_tokens, max_concurrency = get_kv_cache_capacity( - vllm_config, scheduler_kv_cache_config - ) - vllm_config.cache_config.kv_cache_size_tokens = num_tokens - vllm_config.cache_config.kv_cache_max_concurrency = max_concurrency + update_kv_cache_capacity(vllm_config, scheduler_kv_cache_config) vllm_config.validate_block_size()