[Metrics] Add group-aware KV cache capacity to vllm:cache_config_info (#42206)
The startup log already reports the correct group-aware KV cache capacity for hybrid models, but Prometheus did not expose matching info in 'vllm:cache_config_info`. This PR adds kv_cache_size_tokens and kv_cache_max_concurrency. Signed-off-by: Ethan Feng <ethan.fengch@gmail.com>
This commit is contained in:
@@ -28,6 +28,7 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
estimate_max_model_len,
|
||||
generate_block_hash_extra_keys,
|
||||
generate_scheduler_kv_cache_config,
|
||||
get_kv_cache_capacity,
|
||||
get_kv_cache_configs,
|
||||
get_max_concurrency_for_kv_cache_config,
|
||||
get_request_block_hasher,
|
||||
@@ -1459,6 +1460,11 @@ def test_get_max_concurrency_for_kv_cache_config():
|
||||
vllm_config, kv_cache_config_hybrid_model
|
||||
)
|
||||
assert max_concurrency_hybrid_model == 3
|
||||
num_tokens, max_concurrency = get_kv_cache_capacity(
|
||||
vllm_config, kv_cache_config_hybrid_model
|
||||
)
|
||||
assert num_tokens == max_concurrency_hybrid_model * max_model_len
|
||||
assert max_concurrency == max_concurrency_hybrid_model
|
||||
|
||||
|
||||
def test_allocate_with_lookahead():
|
||||
|
||||
Reference in New Issue
Block a user