[Core][Feat] Pluggable KVCacheConfigBuilder for platform/model-specific KV cache planning (#53558)

Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
wangxiyuan
2026-10-16 23:16:47 +00:00
committed by GitHub
co-authored by Claude Opus 5.5 Yifan Qiao
parent 2268d83bd9
commit b544587408
9 changed files with 675 additions and 49 deletions
+9 -13
View File
@@ -1036,26 +1036,22 @@ def _extrapolate_full_graph_memory(mem_samples: list[int], total_graphs: int) ->
def _init_minimal_kv_cache_for_profiling(runner: "GPUModelRunner") -> None:
"""Allocate the smallest KV cache that still lets every graph be captured."""
from vllm.v1.core.kv_cache_planning import (
get_kv_cache_config_from_groups,
get_kv_cache_groups,
)
from vllm.v1.core.kv_cache_config_builder import get_kv_cache_config_builder
from vllm.v1.core.kv_cache_planning import get_kv_cache_groups_from_builder
kv_cache_spec = runner.get_kv_cache_spec()
kv_cache_groups = get_kv_cache_groups(runner.vllm_config, kv_cache_spec)
builder = get_kv_cache_config_builder(runner.vllm_config)
kv_cache_groups = get_kv_cache_groups_from_builder(
builder, runner.vllm_config, kv_cache_spec
)
# At least one block per sequence is required to capture the graphs.
min_blocks = (
min(runner.max_num_reqs, runner.compilation_config.max_cudagraph_capture_size)
or 1
)
saved_override = runner.cache_config.num_gpu_blocks_override
runner.cache_config.num_gpu_blocks_override = min_blocks
try:
minimal_config = get_kv_cache_config_from_groups(
runner.vllm_config, kv_cache_groups, available_memory=0
)
finally:
runner.cache_config.num_gpu_blocks_override = saved_override
minimal_config = builder.get_kv_cache_config_from_groups(
runner.vllm_config, kv_cache_groups, num_blocks=min_blocks
)
runner.initialize_kv_cache(minimal_config, is_profiling=True)
runner.cache_config.num_gpu_blocks = minimal_config.num_blocks