[Bugfix] Count per-group blocks in get_max_concurrency_for_kv_cache_config (#48317)

Signed-off-by: David Orman <ormandj@corenode.com>
Co-authored-by: Luke Alonso <lalonso@gmail.com>
Co-authored-by: Martin Vit <martin@voipmonitor.org>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
ormandj
2026-07-21 00:29:10 +00:00
committed by GitHub
co-authored by Luke Alonso Martin Vit Claude Yifan Qiao
parent af91f4b3e4
commit 2e2e626b40
2 changed files with 138 additions and 11 deletions
+15 -11
View File
@@ -939,19 +939,23 @@ def get_max_concurrency_for_kv_cache_config(
) -> float:
"""
Get the maximum concurrency for the given KV cache configuration.
A request at max_model_len consumes whole blocks from each group's block
table — cdiv(per-request bytes, page bytes) of the group's spec — and all
groups draw those block ids from one shared pool, so the per-request
total is the sum over groups. The memory/page ratio is identical whether
a group carries an aggregated UniformTypeKVCacheSpecs (worker config) or
a representative per-layer spec (scheduler config), so both capacity
call sites agree.
"""
num_layer_per_group = max(
len(group.layer_names) for group in kv_cache_config.kv_cache_groups
num_blocks_per_request = sum(
cdiv(
group.kv_cache_spec.max_memory_usage_bytes(vllm_config),
group.kv_cache_spec.page_size_bytes,
)
for group in kv_cache_config.kv_cache_groups
)
max_memory_usage_per_request = num_layer_per_group * max_memory_usage_bytes(
vllm_config, (group.kv_cache_spec for group in kv_cache_config.kv_cache_groups)
)
memory_per_block = (
kv_cache_config.kv_cache_groups[0].kv_cache_spec.page_size_bytes
* num_layer_per_group
)
num_block_per_request = cdiv(max_memory_usage_per_request, memory_per_block)
max_concurrency = kv_cache_config.num_blocks / num_block_per_request
max_concurrency = kv_cache_config.num_blocks / num_blocks_per_request
return max_concurrency