[ROCm][Feature] Support KV connectors with ROCM_AITER_UNIFIED_ATTN (#53695)
Signed-off-by: simondanielsson <simon.danielsson99@hotmail.com> Signed-off-by: Simon Danielsson <70206058+simondanielsson@users.noreply.github.com> Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
co-authored by
coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
Shanshan Shen
parent
e6cb56337b
commit
8359e15aae
@@ -4,6 +4,7 @@
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
resolve_dcp_kv_block_size,
|
||||
resolve_kv_cache_block_sizes,
|
||||
@@ -90,8 +91,16 @@ def build_offloading_config(
|
||||
)
|
||||
|
||||
tokens_per_block = unique_tokens_per_block.pop()
|
||||
assert tokens_per_chunk_int % tokens_per_block == 0
|
||||
blocks_per_chunk = tokens_per_chunk_int // tokens_per_block
|
||||
if tokens_per_chunk_int % tokens_per_block == 0:
|
||||
blocks_per_chunk = tokens_per_chunk_int // tokens_per_block
|
||||
else:
|
||||
raise ValueError(
|
||||
f"'block_size'={tokens_per_chunk_int} in kv_connector_extra_config "
|
||||
f"must be a multiple of the GPU KV cache block size "
|
||||
f"({tokens_per_block} tokens). Use "
|
||||
f"{round_up(tokens_per_chunk_int, tokens_per_block)} instead, or set "
|
||||
f"'blocks_per_chunk' to express the chunk size in blocks."
|
||||
)
|
||||
|
||||
worker_kv_bytes_per_block = 0
|
||||
if kv_cache_config.num_blocks > 0 and kv_cache_config.kv_cache_tensors:
|
||||
|
||||
@@ -8,6 +8,7 @@ import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import get_current_vllm_config_or_none
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
@@ -32,6 +33,14 @@ from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheLayout
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _kv_connector_enabled() -> bool:
|
||||
vllm_config = get_current_vllm_config_or_none()
|
||||
if vllm_config is None:
|
||||
return False
|
||||
kv_transfer_config = vllm_config.kv_transfer_config
|
||||
return kv_transfer_config is not None and kv_transfer_config.is_kv_transfer_instance
|
||||
|
||||
|
||||
class RocmAiterUnifiedAttentionMetadataBuilder(RocmAttentionMetadataBuilder):
|
||||
def build_for_cudagraph_capture(
|
||||
self, common_attn_metadata: CommonAttentionMetadata
|
||||
@@ -84,6 +93,10 @@ class RocmAiterUnifiedAttentionBackend(RocmAttentionBackend):
|
||||
def supports_non_causal(cls) -> bool:
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def supports_kv_connector(cls) -> bool:
|
||||
return True
|
||||
|
||||
forward_includes_kv_cache_update: bool = False
|
||||
|
||||
@staticmethod
|
||||
@@ -110,6 +123,9 @@ class RocmAiterUnifiedAttentionBackend(RocmAttentionBackend):
|
||||
# K and V come out of the content dim as transposed views rather than
|
||||
# copies, so the head dim may sit on either side of the block dim, but
|
||||
# the layer must stay outermost.
|
||||
if _kv_connector_enabled():
|
||||
# Connectors like MoRI assume contiguous blocks
|
||||
return (KVCacheLayout.LBHNC,)
|
||||
return (KVCacheLayout.LBHNC, KVCacheLayout.LHBNC)
|
||||
|
||||
@staticmethod
|
||||
|
||||
Reference in New Issue
Block a user