[ROCm][Feature] Support KV connectors with ROCM_AITER_UNIFIED_ATTN (#53695)

Signed-off-by: simondanielsson <simon.danielsson99@hotmail.com>
Signed-off-by: Simon Danielsson <70206058+simondanielsson@users.noreply.github.com>
Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
Simon Danielsson
2026-09-10 12:48:42 -05:00
committed by GitHub
co-authored by coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Shanshan Shen
parent e6cb56337b
commit 8359e15aae
4 changed files with 154 additions and 3 deletions
@@ -455,3 +455,115 @@ def test_sparse_not_supported(mock_vllm_config):
RocmPlatform.get_attn_backend_cls(
selected_backend=None, attn_selector_config=attn_selector_config
)
def _kv_connector_selector_config() -> AttentionSelectorConfig:
return AttentionSelectorConfig(
head_size=128,
dtype=torch.float16,
kv_cache_dtype="auto",
block_size=16,
use_mla=False,
has_sink=False,
use_sparse=False,
use_kv_connector=True,
)
def test_unified_attn_declares_kv_connector_support():
"""ROCM_AITER_UNIFIED_ATTN opts into KV connectors and ROCM_ATTN does not."""
from vllm.v1.attention.backends.rocm_aiter_unified_attn import (
RocmAiterUnifiedAttentionBackend,
)
from vllm.v1.attention.backends.rocm_attn import RocmAttentionBackend
assert RocmAiterUnifiedAttentionBackend.supports_kv_connector() is True
assert RocmAttentionBackend.supports_kv_connector() is False
def test_unified_attn_supports_kv_connector(mock_vllm_config, mock_get_cdna_version):
"""ROCM_AITER_UNIFIED_ATTN can be selected with KV connectors."""
from vllm.platforms.rocm import RocmPlatform
backend_path = RocmPlatform.get_attn_backend_cls(
selected_backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
attn_selector_config=_kv_connector_selector_config(),
)
assert backend_path == AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN.get_path()
def test_rocm_attn_rejects_kv_connector(mock_vllm_config, mock_get_cdna_version):
"""Selecting ROCM_ATTN with a KV connector is illegal."""
from vllm.platforms.rocm import RocmPlatform
attn_selector_config = _kv_connector_selector_config()
with pytest.raises(ValueError, match="KV connector not supported"):
RocmPlatform.get_attn_backend_cls(
selected_backend=AttentionBackendEnum.ROCM_ATTN,
attn_selector_config=attn_selector_config,
)
@pytest.mark.parametrize(
"aiter_found, expected_backend",
[
(True, AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN),
(False, AttentionBackendEnum.TRITON_ATTN),
],
)
def test_auto_selection_for_kv_connector(
aiter_found, expected_backend, mock_vllm_config, mock_get_cdna_version
):
"""Auto-selection with a KV connector and AITER enabled resolves to unified attn,
and to triton attn if AITER not enabled."""
from vllm.platforms.rocm import RocmPlatform
with patch(
"vllm._aiter_ops.is_aiter_found_and_supported", return_value=aiter_found
):
backend_path = RocmPlatform.get_attn_backend_cls(
selected_backend=None,
attn_selector_config=_kv_connector_selector_config(),
)
assert backend_path == expected_backend.get_path()
def test_unified_attn_prefers_block_contiguous_layout():
"""Unified attn prefers a block-first KV layout, hence ok with kv connectors."""
from vllm.v1.attention.backends.rocm_aiter_unified_attn import (
RocmAiterUnifiedAttentionBackend,
)
from vllm.v1.attention.backends.rocm_attn import RocmAttentionBackend
unified_preferred = RocmAiterUnifiedAttentionBackend.supported_kv_cache_layouts()[0]
rocm_attn_preferred = RocmAttentionBackend.supported_kv_cache_layouts()[0]
assert unified_preferred.is_block_contiguous is True
assert rocm_attn_preferred.is_block_contiguous is False
def test_unified_attn_drops_lhbnc_with_kv_connector():
"""Connectors move a block as one contiguous byte range, which LHBNC breaks."""
from vllm.config import KVTransferConfig, VllmConfig, set_current_vllm_config
from vllm.v1.attention.backends.rocm_aiter_unified_attn import (
RocmAiterUnifiedAttentionBackend,
)
from vllm.v1.kv_cache_interface import KVCacheLayout
assert KVCacheLayout.LHBNC in (
RocmAiterUnifiedAttentionBackend.supported_kv_cache_layouts()
)
config = VllmConfig(
kv_transfer_config=KVTransferConfig(
kv_connector="ExampleConnector", kv_role="kv_both"
)
)
with set_current_vllm_config(config):
layouts = RocmAiterUnifiedAttentionBackend.supported_kv_cache_layouts()
assert layouts
assert all(layout.is_block_compact for layout in layouts)
@@ -15,12 +15,26 @@ from vllm.config import KVEventsConfig, KVTransferConfig
from vllm.distributed.kv_events import BlockStored, KVEventBatch
from vllm.platforms import current_platform
CPU_BLOCK_SIZES: int = 64 if current_platform.is_xpu() else 48
# The offloaded block size must stay a whole number of GPU blocks.
CPU_BLOCK_SIZES: int
if current_platform.is_xpu():
CPU_BLOCK_SIZES = 64
elif current_platform.is_rocm():
# ROCM_AITER_UNIFIED_ATTN prefers 64-token GPU blocks, so 192 covers both that
# and the 16-token default.
CPU_BLOCK_SIZES = 192
else:
CPU_BLOCK_SIZES = 48
_ATTN_BACKENDS: list[str] = []
if current_platform.is_cuda():
_ATTN_BACKENDS = ["FLASH_ATTN", "FLASHINFER", "TRITON_ATTN"]
elif current_platform.is_rocm():
from vllm._aiter_ops import is_aiter_found_and_supported
_ATTN_BACKENDS = ["TRITON_ATTN"]
if is_aiter_found_and_supported():
_ATTN_BACKENDS.append("ROCM_AITER_UNIFIED_ATTN")
elif current_platform.is_xpu():
_ATTN_BACKENDS = ["FLASH_ATTN", "TRITON_ATTN"]