[Metrics] Add group-aware KV cache capacity to vllm:cache_config_info (#42206)

The startup log already reports the correct group-aware KV cache capacity for
hybrid models, but Prometheus did not expose matching info in 'vllm:cache_config_info`.

This PR adds kv_cache_size_tokens and kv_cache_max_concurrency.

Signed-off-by: Ethan Feng <ethan.fengch@gmail.com>
This commit is contained in:
Ethan Feng
2026-06-12 11:49:44 +00:00
committed by GitHub
parent 8af550b399
commit b7f9b6ab27
7 changed files with 75 additions and 26 deletions
@@ -289,6 +289,17 @@ async def test_metrics_exist(
continue continue
assert metric in response.text assert metric in response.text
cache_config_samples = [
sample
for family in text_string_to_metric_families(response.text)
if family.name == "vllm:cache_config_info"
for sample in family.samples
]
assert cache_config_samples
for sample in cache_config_samples:
assert sample.labels.get("kv_cache_size_tokens") not in (None, "None", "")
assert sample.labels.get("kv_cache_max_concurrency") not in (None, "None", "")
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_abort_metrics_reset( async def test_abort_metrics_reset(
+6
View File
@@ -28,6 +28,7 @@ from vllm.v1.core.kv_cache_utils import (
estimate_max_model_len, estimate_max_model_len,
generate_block_hash_extra_keys, generate_block_hash_extra_keys,
generate_scheduler_kv_cache_config, generate_scheduler_kv_cache_config,
get_kv_cache_capacity,
get_kv_cache_configs, get_kv_cache_configs,
get_max_concurrency_for_kv_cache_config, get_max_concurrency_for_kv_cache_config,
get_request_block_hasher, get_request_block_hasher,
@@ -1459,6 +1460,11 @@ def test_get_max_concurrency_for_kv_cache_config():
vllm_config, kv_cache_config_hybrid_model vllm_config, kv_cache_config_hybrid_model
) )
assert max_concurrency_hybrid_model == 3 assert max_concurrency_hybrid_model == 3
num_tokens, max_concurrency = get_kv_cache_capacity(
vllm_config, kv_cache_config_hybrid_model
)
assert num_tokens == max_concurrency_hybrid_model * max_model_len
assert max_concurrency == max_concurrency_hybrid_model
def test_allocate_with_lookahead(): def test_allocate_with_lookahead():
+10
View File
@@ -146,6 +146,14 @@ class CacheConfig:
num_cpu_blocks: int | None = field(default=None, init=False) num_cpu_blocks: int | None = field(default=None, init=False)
"""The number of blocks to allocate for CPU memory.""" """The number of blocks to allocate for CPU memory."""
# Set after KV cache initialization.
kv_cache_size_tokens: int | None = field(default=None, init=False)
"""Per-DP-engine KV cache capacity in tokens (group-aware). Uses
group-aware capacity since num_gpu_blocks * block_size can be wrong
for hybrid models where requests occupy multiple KV cache groups."""
kv_cache_max_concurrency: float | None = field(default=None, init=False)
"""Per-DP-engine maximum concurrency at max_model_len tokens."""
kv_sharing_fast_prefill: bool = False kv_sharing_fast_prefill: bool = False
"""This feature is work in progress and no prefill optimization takes place """This feature is work in progress and no prefill optimization takes place
with this flag enabled currently. with this flag enabled currently.
@@ -204,6 +212,8 @@ class CacheConfig:
# Post-init/derived counters # Post-init/derived counters
"num_gpu_blocks", "num_gpu_blocks",
"num_cpu_blocks", "num_cpu_blocks",
"kv_cache_size_tokens",
"kv_cache_max_concurrency",
# WIP feature toggle not impacting compiled graph shape # WIP feature toggle not impacting compiled graph shape
"kv_sharing_fast_prefill", "kv_sharing_fast_prefill",
} }
+19 -24
View File
@@ -1717,36 +1717,17 @@ def generate_scheduler_kv_cache_config(
return cfg return cfg
def _report_kv_cache_config( def get_kv_cache_capacity(
vllm_config: VllmConfig, kv_cache_config: KVCacheConfig vllm_config: VllmConfig, kv_cache_config: KVCacheConfig
) -> None: ) -> tuple[int, float]:
""" """
Log resolved KV cache configuration. Get the group-aware KV cache token capacity and max concurrency.
Args:
vllm_config: The global VllmConfig
kv_cache_config: The resolved KV cache configuration
""" """
max_model_len = vllm_config.model_config.max_model_len max_model_len = vllm_config.model_config.max_model_len
max_concurrency = get_max_concurrency_for_kv_cache_config( max_concurrency = get_max_concurrency_for_kv_cache_config(
vllm_config, kv_cache_config vllm_config, kv_cache_config
) )
return int(max_concurrency * max_model_len), max_concurrency
# GPU KV cache size in tokens = max_concurrency * max_model_len: the total
# tokens of context the pool can hold at peak utilization. Sourcing this
# from the concurrency calculation handles hybrid layouts correctly: SWA /
# chunked-local groups have a per-request block count that's capped by
# their window, so a naive `num_blocks // num_groups * block_size` formula
# underestimates capacity for these models. DCP/PCP sharding is already
# accounted for in each spec's `max_memory_usage_bytes`.
num_tokens = int(max_concurrency * max_model_len)
logger.info_once("GPU KV cache size: %s tokens", f"{num_tokens:,}")
logger.info_once(
"Maximum concurrency for %s tokens per request: %.2fx",
f"{max_model_len:,}",
max_concurrency,
)
def _max_memory_usage_bytes_from_groups( def _max_memory_usage_bytes_from_groups(
@@ -2085,7 +2066,21 @@ def get_kv_cache_configs(
tensor.size = tensor.size // num_blocks_old * min_num_blocks tensor.size = tensor.size // num_blocks_old * min_num_blocks
if len(kv_cache_config.kv_cache_groups) > 0: if len(kv_cache_config.kv_cache_groups) > 0:
_report_kv_cache_config(vllm_config, kv_cache_config) max_model_len = vllm_config.model_config.max_model_len
# GPU KV cache size in tokens = max_concurrency * max_model_len:
# the total tokens of context the pool can hold at peak
# utilization. Sourcing this from the concurrency calculation
# handles hybrid layouts correctly.
num_tokens, max_concurrency = get_kv_cache_capacity(
vllm_config, kv_cache_config
)
logger.info_once("GPU KV cache size: %s tokens", f"{num_tokens:,}")
logger.info_once(
"Maximum concurrency for %s tokens per request: %.2fx",
f"{max_model_len:,}",
max_concurrency,
)
return kv_cache_configs return kv_cache_configs
+3
View File
@@ -78,6 +78,9 @@ class EngineCoreReadyResponse:
dp_stats_address: str | None dp_stats_address: str | None
dtype: str dtype: str
vllm_version: str vllm_version: str
# KV cache capacity (None for encoder-only/attention-free models).
kv_cache_size_tokens: int | None = None
kv_cache_max_concurrency: float | None = None
class EngineCoreRequest( class EngineCoreRequest(
+12
View File
@@ -45,6 +45,7 @@ from vllm.utils.system_utils import decorate_logs, set_process_title
from vllm.v1.core.kv_cache_utils import ( from vllm.v1.core.kv_cache_utils import (
BlockHash, BlockHash,
generate_scheduler_kv_cache_config, generate_scheduler_kv_cache_config,
get_kv_cache_capacity,
get_kv_cache_configs, get_kv_cache_configs,
get_request_block_hasher, get_request_block_hasher,
init_none_hash, init_none_hash,
@@ -286,6 +287,11 @@ class EngineCore:
vllm_config.cache_config.block_size = min( vllm_config.cache_config.block_size = min(
g.kv_cache_spec.block_size for g in kv_cache_groups g.kv_cache_spec.block_size for g in kv_cache_groups
) )
num_tokens, max_concurrency = get_kv_cache_capacity(
vllm_config, scheduler_kv_cache_config
)
vllm_config.cache_config.kv_cache_size_tokens = num_tokens
vllm_config.cache_config.kv_cache_max_concurrency = max_concurrency
vllm_config.validate_block_size() vllm_config.validate_block_size()
@@ -1494,6 +1500,12 @@ class EngineCoreProc(EngineCore):
dp_stats_address=self.frontend_stats_publish_address, dp_stats_address=self.frontend_stats_publish_address,
dtype=str(self.vllm_config.model_config.dtype).removeprefix("torch."), dtype=str(self.vllm_config.model_config.dtype).removeprefix("torch."),
vllm_version=VLLM_VERSION, vllm_version=VLLM_VERSION,
kv_cache_size_tokens=(
self.vllm_config.cache_config.kv_cache_size_tokens
),
kv_cache_max_concurrency=(
self.vllm_config.cache_config.kv_cache_max_concurrency
),
) )
ready_payload = msgspec.msgpack.encode(ready_response) ready_payload = msgspec.msgpack.encode(ready_response)
for input_socket in input_sockets: for input_socket in input_sockets:
+14 -2
View File
@@ -720,14 +720,26 @@ class MPClient(EngineCoreClient):
) )
# Setup KV cache config with initialization state from # Setup KV cache config with initialization state from
# engine core process. Sum values from all engines in DP case. # engine core process. Sum num_gpu_blocks from all engines in DP case.
num_gpu_blocks = vllm_config.cache_config.num_gpu_blocks or 0 num_gpu_blocks = vllm_config.cache_config.num_gpu_blocks or 0
num_gpu_blocks += response.num_gpu_blocks num_gpu_blocks += response.num_gpu_blocks
vllm_config.cache_config.num_gpu_blocks = num_gpu_blocks vllm_config.cache_config.num_gpu_blocks = num_gpu_blocks
# Sync block_size: may be enlarged by _align_hybrid_block_size in the # Sync block_size: may be enlarged by _align_hybrid_block_size in the
# worker for hybrid Mamba models. # worker for hybrid Mamba models.
vllm_config.cache_config.block_size = response.block_size cache_config = vllm_config.cache_config
cache_config.block_size = response.block_size
# Keep these as per-engine cache_config_info values; do not sum across DP.
cache_config.kv_cache_size_tokens = (
getattr(cache_config, "kv_cache_size_tokens", None)
if getattr(cache_config, "kv_cache_size_tokens", None) is not None
else response.kv_cache_size_tokens
)
cache_config.kv_cache_max_concurrency = (
getattr(cache_config, "kv_cache_max_concurrency", None)
if getattr(cache_config, "kv_cache_max_concurrency", None) is not None
else response.kv_cache_max_concurrency
)
# In external DP LB mode, the coordinator address that the # In external DP LB mode, the coordinator address that the
# front-end procs connect to is obtained by each engine via it's # front-end procs connect to is obtained by each engine via it's