[Bugfix][Core] Log KV cache capacity after block-size resolution (#50462)
Signed-off-by: Tanmay Dixit <22117787+tandixit95@users.noreply.github.com> Signed-off-by: Michael Goin <mgoin64@gmail.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Tanmay Dixit <22117787+tandixit95@users.noreply.github.com> Co-authored-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Tanmay Dixit
Michael Goin
Matthew Bonanni
Claude Opus 5
parent
5789897aa4
commit
5f2ee2fa8c
@@ -1853,6 +1853,23 @@ def get_kv_cache_capacity(
|
|||||||
return int(max_concurrency * max_model_len), max_concurrency
|
return int(max_concurrency * max_model_len), max_concurrency
|
||||||
|
|
||||||
|
|
||||||
|
def update_kv_cache_capacity(
|
||||||
|
vllm_config: VllmConfig, kv_cache_config: KVCacheConfig
|
||||||
|
) -> None:
|
||||||
|
"""Store and log the resolved KV cache capacity."""
|
||||||
|
num_tokens, max_concurrency = get_kv_cache_capacity(vllm_config, kv_cache_config)
|
||||||
|
vllm_config.cache_config.kv_cache_size_tokens = num_tokens
|
||||||
|
vllm_config.cache_config.kv_cache_max_concurrency = max_concurrency
|
||||||
|
max_model_len = vllm_config.model_config.max_model_len
|
||||||
|
logger.info_once(
|
||||||
|
"GPU KV cache size: %s tokens, "
|
||||||
|
"Maximum concurrency for %s tokens per request: %.2fx",
|
||||||
|
f"{num_tokens:,}",
|
||||||
|
f"{max_model_len:,}",
|
||||||
|
max_concurrency,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _max_memory_usage_bytes_from_groups(
|
def _max_memory_usage_bytes_from_groups(
|
||||||
vllm_config: VllmConfig,
|
vllm_config: VllmConfig,
|
||||||
kv_cache_groups: list[KVCacheGroupSpec],
|
kv_cache_groups: list[KVCacheGroupSpec],
|
||||||
@@ -2188,23 +2205,6 @@ def get_kv_cache_configs(
|
|||||||
assert tensor.size % num_blocks_old == 0
|
assert tensor.size % num_blocks_old == 0
|
||||||
tensor.size = tensor.size // num_blocks_old * min_num_blocks
|
tensor.size = tensor.size // num_blocks_old * min_num_blocks
|
||||||
|
|
||||||
if len(kv_cache_config.kv_cache_groups) > 0:
|
|
||||||
max_model_len = vllm_config.model_config.max_model_len
|
|
||||||
# GPU KV cache size in tokens = max_concurrency * max_model_len:
|
|
||||||
# the total tokens of context the pool can hold at peak
|
|
||||||
# utilization. Sourcing this from the concurrency calculation
|
|
||||||
# handles hybrid layouts correctly.
|
|
||||||
num_tokens, max_concurrency = get_kv_cache_capacity(
|
|
||||||
vllm_config, kv_cache_config
|
|
||||||
)
|
|
||||||
|
|
||||||
logger.info_once("GPU KV cache size: %s tokens", f"{num_tokens:,}")
|
|
||||||
logger.info_once(
|
|
||||||
"Maximum concurrency for %s tokens per request: %.2fx",
|
|
||||||
f"{max_model_len:,}",
|
|
||||||
max_concurrency,
|
|
||||||
)
|
|
||||||
|
|
||||||
return kv_cache_configs
|
return kv_cache_configs
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -46,11 +46,11 @@ from vllm.utils.system_utils import decorate_logs, set_process_title
|
|||||||
from vllm.v1.core.kv_cache_utils import (
|
from vllm.v1.core.kv_cache_utils import (
|
||||||
BlockHash,
|
BlockHash,
|
||||||
generate_scheduler_kv_cache_config,
|
generate_scheduler_kv_cache_config,
|
||||||
get_kv_cache_capacity,
|
|
||||||
get_kv_cache_configs,
|
get_kv_cache_configs,
|
||||||
get_request_block_hasher,
|
get_request_block_hasher,
|
||||||
init_none_hash,
|
init_none_hash,
|
||||||
resolve_kv_cache_block_sizes,
|
resolve_kv_cache_block_sizes,
|
||||||
|
update_kv_cache_capacity,
|
||||||
)
|
)
|
||||||
from vllm.v1.core.sched.interface import PauseState, SchedulerInterface
|
from vllm.v1.core.sched.interface import PauseState, SchedulerInterface
|
||||||
from vllm.v1.core.sched.output import SchedulerOutput
|
from vllm.v1.core.sched.output import SchedulerOutput
|
||||||
@@ -319,11 +319,7 @@ class EngineCore:
|
|||||||
vllm_config.cache_config.block_size = min(
|
vllm_config.cache_config.block_size = min(
|
||||||
g.kv_cache_spec.block_size for g in kv_cache_groups
|
g.kv_cache_spec.block_size for g in kv_cache_groups
|
||||||
)
|
)
|
||||||
num_tokens, max_concurrency = get_kv_cache_capacity(
|
update_kv_cache_capacity(vllm_config, scheduler_kv_cache_config)
|
||||||
vllm_config, scheduler_kv_cache_config
|
|
||||||
)
|
|
||||||
vllm_config.cache_config.kv_cache_size_tokens = num_tokens
|
|
||||||
vllm_config.cache_config.kv_cache_max_concurrency = max_concurrency
|
|
||||||
|
|
||||||
vllm_config.validate_block_size()
|
vllm_config.validate_block_size()
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user