[ROCm][Feature] Support KV connectors with ROCM_AITER_UNIFIED_ATTN (#53695)
Signed-off-by: simondanielsson <simon.danielsson99@hotmail.com> Signed-off-by: Simon Danielsson <70206058+simondanielsson@users.noreply.github.com> Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
co-authored by
coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
Shanshan Shen
parent
e6cb56337b
commit
8359e15aae
@@ -4,6 +4,7 @@
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
resolve_dcp_kv_block_size,
|
||||
resolve_kv_cache_block_sizes,
|
||||
@@ -90,8 +91,16 @@ def build_offloading_config(
|
||||
)
|
||||
|
||||
tokens_per_block = unique_tokens_per_block.pop()
|
||||
assert tokens_per_chunk_int % tokens_per_block == 0
|
||||
blocks_per_chunk = tokens_per_chunk_int // tokens_per_block
|
||||
if tokens_per_chunk_int % tokens_per_block == 0:
|
||||
blocks_per_chunk = tokens_per_chunk_int // tokens_per_block
|
||||
else:
|
||||
raise ValueError(
|
||||
f"'block_size'={tokens_per_chunk_int} in kv_connector_extra_config "
|
||||
f"must be a multiple of the GPU KV cache block size "
|
||||
f"({tokens_per_block} tokens). Use "
|
||||
f"{round_up(tokens_per_chunk_int, tokens_per_block)} instead, or set "
|
||||
f"'blocks_per_chunk' to express the chunk size in blocks."
|
||||
)
|
||||
|
||||
worker_kv_bytes_per_block = 0
|
||||
if kv_cache_config.num_blocks > 0 and kv_cache_config.kv_cache_tensors:
|
||||
|
||||
Reference in New Issue
Block a user