Add blocks_per_chunk configuration for KV offloading to support heterogeneous KV cache groups (#48878)

Signed-off-by: Debasish-87 <22btics06@suiit.ac.in>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Debasish Mohanty
2026-07-17 09:00:12 +03:00
committed by GitHub
co-authored by Or Ozeri
parent 3b6c96a101
commit 472d330c21
2 changed files with 55 additions and 1 deletions
@@ -58,15 +58,32 @@ def build_offloading_config(
)
blocks_per_chunk = 1
blocks_per_chunk_config = extra_config.get("blocks_per_chunk")
tokens_per_chunk = extra_config.get("block_size")
if tokens_per_chunk is not None:
if blocks_per_chunk_config is not None and tokens_per_chunk is not None:
raise ValueError(
"Specify only one of 'block_size' or 'blocks_per_chunk' "
"in kv_connector_extra_config."
)
if blocks_per_chunk_config is not None:
blocks_per_chunk = int(blocks_per_chunk_config)
if blocks_per_chunk <= 0:
raise ValueError("'blocks_per_chunk' must be greater than 0.")
elif tokens_per_chunk is not None:
tokens_per_chunk_int = int(tokens_per_chunk)
unique_tokens_per_block = {group.tokens_per_block for group in groups}
assert len(unique_tokens_per_block) == 1, (
"If 'block_size' is specified in kv_connector_extra_config, "
"there must be at least one KV cache group, "
"and all groups must have the same block size."
)
tokens_per_block = unique_tokens_per_block.pop()
assert tokens_per_chunk_int % tokens_per_block == 0
blocks_per_chunk = tokens_per_chunk_int // tokens_per_block