[Metrics] Add group-aware KV cache capacity to vllm:cache_config_info (#42206)
The startup log already reports the correct group-aware KV cache capacity for hybrid models, but Prometheus did not expose matching info in 'vllm:cache_config_info`. This PR adds kv_cache_size_tokens and kv_cache_max_concurrency. Signed-off-by: Ethan Feng <ethan.fengch@gmail.com>
This commit is contained in:
@@ -78,6 +78,9 @@ class EngineCoreReadyResponse:
|
||||
dp_stats_address: str | None
|
||||
dtype: str
|
||||
vllm_version: str
|
||||
# KV cache capacity (None for encoder-only/attention-free models).
|
||||
kv_cache_size_tokens: int | None = None
|
||||
kv_cache_max_concurrency: float | None = None
|
||||
|
||||
|
||||
class EngineCoreRequest(
|
||||
|
||||
@@ -45,6 +45,7 @@ from vllm.utils.system_utils import decorate_logs, set_process_title
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
generate_scheduler_kv_cache_config,
|
||||
get_kv_cache_capacity,
|
||||
get_kv_cache_configs,
|
||||
get_request_block_hasher,
|
||||
init_none_hash,
|
||||
@@ -286,6 +287,11 @@ class EngineCore:
|
||||
vllm_config.cache_config.block_size = min(
|
||||
g.kv_cache_spec.block_size for g in kv_cache_groups
|
||||
)
|
||||
num_tokens, max_concurrency = get_kv_cache_capacity(
|
||||
vllm_config, scheduler_kv_cache_config
|
||||
)
|
||||
vllm_config.cache_config.kv_cache_size_tokens = num_tokens
|
||||
vllm_config.cache_config.kv_cache_max_concurrency = max_concurrency
|
||||
|
||||
vllm_config.validate_block_size()
|
||||
|
||||
@@ -1494,6 +1500,12 @@ class EngineCoreProc(EngineCore):
|
||||
dp_stats_address=self.frontend_stats_publish_address,
|
||||
dtype=str(self.vllm_config.model_config.dtype).removeprefix("torch."),
|
||||
vllm_version=VLLM_VERSION,
|
||||
kv_cache_size_tokens=(
|
||||
self.vllm_config.cache_config.kv_cache_size_tokens
|
||||
),
|
||||
kv_cache_max_concurrency=(
|
||||
self.vllm_config.cache_config.kv_cache_max_concurrency
|
||||
),
|
||||
)
|
||||
ready_payload = msgspec.msgpack.encode(ready_response)
|
||||
for input_socket in input_sockets:
|
||||
|
||||
@@ -720,14 +720,26 @@ class MPClient(EngineCoreClient):
|
||||
)
|
||||
|
||||
# Setup KV cache config with initialization state from
|
||||
# engine core process. Sum values from all engines in DP case.
|
||||
# engine core process. Sum num_gpu_blocks from all engines in DP case.
|
||||
num_gpu_blocks = vllm_config.cache_config.num_gpu_blocks or 0
|
||||
num_gpu_blocks += response.num_gpu_blocks
|
||||
vllm_config.cache_config.num_gpu_blocks = num_gpu_blocks
|
||||
|
||||
# Sync block_size: may be enlarged by _align_hybrid_block_size in the
|
||||
# worker for hybrid Mamba models.
|
||||
vllm_config.cache_config.block_size = response.block_size
|
||||
cache_config = vllm_config.cache_config
|
||||
cache_config.block_size = response.block_size
|
||||
# Keep these as per-engine cache_config_info values; do not sum across DP.
|
||||
cache_config.kv_cache_size_tokens = (
|
||||
getattr(cache_config, "kv_cache_size_tokens", None)
|
||||
if getattr(cache_config, "kv_cache_size_tokens", None) is not None
|
||||
else response.kv_cache_size_tokens
|
||||
)
|
||||
cache_config.kv_cache_max_concurrency = (
|
||||
getattr(cache_config, "kv_cache_max_concurrency", None)
|
||||
if getattr(cache_config, "kv_cache_max_concurrency", None) is not None
|
||||
else response.kv_cache_max_concurrency
|
||||
)
|
||||
|
||||
# In external DP LB mode, the coordinator address that the
|
||||
# front-end procs connect to is obtained by each engine via it's
|
||||
|
||||
Reference in New Issue
Block a user