Add blocks_per_chunk configuration for KV offloading to support heterogeneous KV cache groups (#48878)
Signed-off-by: Debasish-87 <22btics06@suiit.ac.in> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
Or Ozeri
parent
3b6c96a101
commit
472d330c21
@@ -58,15 +58,32 @@ def build_offloading_config(
|
||||
)
|
||||
|
||||
blocks_per_chunk = 1
|
||||
blocks_per_chunk_config = extra_config.get("blocks_per_chunk")
|
||||
tokens_per_chunk = extra_config.get("block_size")
|
||||
if tokens_per_chunk is not None:
|
||||
|
||||
if blocks_per_chunk_config is not None and tokens_per_chunk is not None:
|
||||
raise ValueError(
|
||||
"Specify only one of 'block_size' or 'blocks_per_chunk' "
|
||||
"in kv_connector_extra_config."
|
||||
)
|
||||
|
||||
if blocks_per_chunk_config is not None:
|
||||
blocks_per_chunk = int(blocks_per_chunk_config)
|
||||
|
||||
if blocks_per_chunk <= 0:
|
||||
raise ValueError("'blocks_per_chunk' must be greater than 0.")
|
||||
|
||||
elif tokens_per_chunk is not None:
|
||||
tokens_per_chunk_int = int(tokens_per_chunk)
|
||||
|
||||
unique_tokens_per_block = {group.tokens_per_block for group in groups}
|
||||
|
||||
assert len(unique_tokens_per_block) == 1, (
|
||||
"If 'block_size' is specified in kv_connector_extra_config, "
|
||||
"there must be at least one KV cache group, "
|
||||
"and all groups must have the same block size."
|
||||
)
|
||||
|
||||
tokens_per_block = unique_tokens_per_block.pop()
|
||||
assert tokens_per_chunk_int % tokens_per_block == 0
|
||||
blocks_per_chunk = tokens_per_chunk_int // tokens_per_block
|
||||
|
||||
Reference in New Issue
Block a user