[Bugfix] Count per-group blocks in get_max_concurrency_for_kv_cache_config (#48317)
Signed-off-by: David Orman <ormandj@corenode.com> Co-authored-by: Luke Alonso <lalonso@gmail.com> Co-authored-by: Martin Vit <martin@voipmonitor.org> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
Luke Alonso
Martin Vit
Claude
Yifan Qiao
parent
af91f4b3e4
commit
2e2e626b40
@@ -939,19 +939,23 @@ def get_max_concurrency_for_kv_cache_config(
|
||||
) -> float:
|
||||
"""
|
||||
Get the maximum concurrency for the given KV cache configuration.
|
||||
|
||||
A request at max_model_len consumes whole blocks from each group's block
|
||||
table — cdiv(per-request bytes, page bytes) of the group's spec — and all
|
||||
groups draw those block ids from one shared pool, so the per-request
|
||||
total is the sum over groups. The memory/page ratio is identical whether
|
||||
a group carries an aggregated UniformTypeKVCacheSpecs (worker config) or
|
||||
a representative per-layer spec (scheduler config), so both capacity
|
||||
call sites agree.
|
||||
"""
|
||||
num_layer_per_group = max(
|
||||
len(group.layer_names) for group in kv_cache_config.kv_cache_groups
|
||||
num_blocks_per_request = sum(
|
||||
cdiv(
|
||||
group.kv_cache_spec.max_memory_usage_bytes(vllm_config),
|
||||
group.kv_cache_spec.page_size_bytes,
|
||||
)
|
||||
for group in kv_cache_config.kv_cache_groups
|
||||
)
|
||||
max_memory_usage_per_request = num_layer_per_group * max_memory_usage_bytes(
|
||||
vllm_config, (group.kv_cache_spec for group in kv_cache_config.kv_cache_groups)
|
||||
)
|
||||
memory_per_block = (
|
||||
kv_cache_config.kv_cache_groups[0].kv_cache_spec.page_size_bytes
|
||||
* num_layer_per_group
|
||||
)
|
||||
num_block_per_request = cdiv(max_memory_usage_per_request, memory_per_block)
|
||||
max_concurrency = kv_cache_config.num_blocks / num_block_per_request
|
||||
max_concurrency = kv_cache_config.num_blocks / num_blocks_per_request
|
||||
return max_concurrency
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user