[ROCm][Feature] Support KV connectors with ROCM_AITER_UNIFIED_ATTN (#53695)

Signed-off-by: simondanielsson <simon.danielsson99@hotmail.com>
Signed-off-by: Simon Danielsson <70206058+simondanielsson@users.noreply.github.com>
Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
Simon Danielsson
2026-09-10 12:48:42 -05:00
committed by GitHub
co-authored by coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Shanshan Shen
parent e6cb56337b
commit 8359e15aae
4 changed files with 154 additions and 3 deletions
@@ -4,6 +4,7 @@
from typing import TYPE_CHECKING
from vllm.utils.math_utils import round_up
from vllm.v1.core.kv_cache_utils import (
resolve_dcp_kv_block_size,
resolve_kv_cache_block_sizes,
@@ -90,8 +91,16 @@ def build_offloading_config(
)
tokens_per_block = unique_tokens_per_block.pop()
assert tokens_per_chunk_int % tokens_per_block == 0
blocks_per_chunk = tokens_per_chunk_int // tokens_per_block
if tokens_per_chunk_int % tokens_per_block == 0:
blocks_per_chunk = tokens_per_chunk_int // tokens_per_block
else:
raise ValueError(
f"'block_size'={tokens_per_chunk_int} in kv_connector_extra_config "
f"must be a multiple of the GPU KV cache block size "
f"({tokens_per_block} tokens). Use "
f"{round_up(tokens_per_chunk_int, tokens_per_block)} instead, or set "
f"'blocks_per_chunk' to express the chunk size in blocks."
)
worker_kv_bytes_per_block = 0
if kv_cache_config.num_blocks > 0 and kv_cache_config.kv_cache_tensors:
@@ -8,6 +8,7 @@ import torch
from vllm import _custom_ops as ops
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import get_current_vllm_config_or_none
from vllm.config.cache import CacheDType
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.quant_utils import (
@@ -32,6 +33,14 @@ from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheLayout
logger = init_logger(__name__)
def _kv_connector_enabled() -> bool:
vllm_config = get_current_vllm_config_or_none()
if vllm_config is None:
return False
kv_transfer_config = vllm_config.kv_transfer_config
return kv_transfer_config is not None and kv_transfer_config.is_kv_transfer_instance
class RocmAiterUnifiedAttentionMetadataBuilder(RocmAttentionMetadataBuilder):
def build_for_cudagraph_capture(
self, common_attn_metadata: CommonAttentionMetadata
@@ -84,6 +93,10 @@ class RocmAiterUnifiedAttentionBackend(RocmAttentionBackend):
def supports_non_causal(cls) -> bool:
return False
@classmethod
def supports_kv_connector(cls) -> bool:
return True
forward_includes_kv_cache_update: bool = False
@staticmethod
@@ -110,6 +123,9 @@ class RocmAiterUnifiedAttentionBackend(RocmAttentionBackend):
# K and V come out of the content dim as transposed views rather than
# copies, so the head dim may sit on either side of the block dim, but
# the layer must stay outermost.
if _kv_connector_enabled():
# Connectors like MoRI assume contiguous blocks
return (KVCacheLayout.LBHNC,)
return (KVCacheLayout.LBHNC, KVCacheLayout.LHBNC)
@staticmethod