[ROCm] Give turboquant boundary layers a layout-compatible backend (#54988)

Signed-off-by: Stefan Koncarevic <stefan.koncarevic@amd.com>
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
stefankoncarevic
2026-09-23 22:00:07 -05:00
committed by GitHub
co-authored by Andreas Karatzas Codex
parent e6dc16cebd
commit e78e367c6e
3 changed files with 232 additions and 16 deletions
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for attention backend selectors."""
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
@@ -34,6 +35,15 @@ def mock_get_cdna_version():
yield
@pytest.fixture
def cleared_attention_selector_cache():
from vllm.v1.attention.selector import _cached_get_attn_backend
_cached_get_attn_backend.cache_clear()
yield
_cached_get_attn_backend.cache_clear()
def test_aiter_unified_attention_uses_dedicated_metadata_builder():
from vllm.v1.attention.backends.rocm_aiter_unified_attn import (
RocmAiterUnifiedAttentionBackend,
@@ -434,6 +444,139 @@ def test_aiter_fa_requires_mi3xx(mock_vllm_config):
)
@pytest.fixture
def turboquant_run_config():
"""Current config of a run whose KV cache dtype is a turboquant_* preset."""
config = SimpleNamespace(
cache_config=SimpleNamespace(cache_dtype="turboquant_k8v4")
)
with patch(
"vllm.config.get_current_vllm_config_or_none",
return_value=config,
):
yield config
def test_turboquant_boundary_selection_is_not_cached_from_ordinary_run(
cleared_attention_selector_cache,
):
from vllm.config import CacheConfig, VllmConfig, set_current_vllm_config
from vllm.v1.attention.backends.utils import get_supported_kv_cache_layouts
from vllm.v1.attention.selector import get_attn_backend
from vllm.v1.kv_cache_layout import KVCacheLayout
ordinary_config = VllmConfig(cache_config=CacheConfig(cache_dtype="auto"))
turboquant_config = VllmConfig(
cache_config=CacheConfig(cache_dtype="turboquant_k8v4")
)
backend_priorities = [
AttentionBackendEnum.ROCM_ATTN,
AttentionBackendEnum.TRITON_ATTN,
AttentionBackendEnum.TURBOQUANT,
]
with patch(
"vllm.platforms.rocm._get_backend_priorities",
return_value=backend_priorities,
):
with set_current_vllm_config(ordinary_config):
ordinary_backend = get_attn_backend(128, torch.float16, "auto")
with set_current_vllm_config(turboquant_config):
boundary_backend = get_attn_backend(128, torch.float16, "auto")
quantized_backend = get_attn_backend(128, torch.float16, "turboquant_k8v4")
assert ordinary_backend is AttentionBackendEnum.ROCM_ATTN.get_class()
assert quantized_backend is AttentionBackendEnum.TURBOQUANT.get_class()
assert get_supported_kv_cache_layouts([boundary_backend, quantized_backend]) == [
KVCacheLayout.LBNHC
]
def test_turboquant_run_does_not_mask_unsupported_selected_backend(
turboquant_run_config,
):
from vllm.platforms.rocm import RocmPlatform
attn_selector_config = AttentionSelectorConfig(
head_size=128,
dtype=torch.float16,
kv_cache_dtype="auto",
block_size=16,
)
with (
patch("vllm.platforms.rocm.get_cdna_version", return_value=1),
pytest.raises(ValueError, match="compute capability not supported"),
):
RocmPlatform.get_attn_backend_cls(
selected_backend=AttentionBackendEnum.ROCM_AITER_FA,
attn_selector_config=attn_selector_config,
)
def test_turboquant_layout_check_respects_backend_override():
from vllm.platforms.rocm import _shares_layout_with_turboquant
from vllm.v1.kv_cache_layout import KVCacheLayout
boundary_backend = MagicMock()
boundary_backend.supported_kv_cache_layouts.return_value = (KVCacheLayout.LBHNC,)
turboquant_override = MagicMock()
turboquant_override.supported_kv_cache_layouts.return_value = (KVCacheLayout.LBHNC,)
with patch.object(
AttentionBackendEnum.TURBOQUANT,
"get_class",
return_value=turboquant_override,
) as get_turboquant_class:
assert _shares_layout_with_turboquant(boundary_backend)
get_turboquant_class.assert_called_once_with()
@pytest.mark.parametrize(
"selected_backend",
[None, "ROCM_ATTN", "ROCM_AITER_FA", "TURBOQUANT"],
)
def test_turboquant_boundary_layers_share_a_layout(
selected_backend,
turboquant_run_config,
mock_get_cdna_version,
):
"""A turboquant_* run must resolve to backends with a common KV layout.
The boundary layers keep the native dtype and pick their own backend while
every other layer picks TURBOQUANT. Engine startup hard-errors when the two
share no layout, so the boundary layers must not land on a native ROCm or
AITER backend, whichever backend the run asked for.
"""
from vllm.platforms.rocm import RocmPlatform
from vllm.utils.import_utils import resolve_obj_by_qualname
from vllm.v1.attention.backends.utils import get_supported_kv_cache_layouts
backend_enum = (
getattr(AttentionBackendEnum, selected_backend) if selected_backend else None
)
def resolve(kv_cache_dtype):
path = RocmPlatform.get_attn_backend_cls(
selected_backend=backend_enum,
attn_selector_config=AttentionSelectorConfig(
head_size=128,
dtype=torch.float16,
kv_cache_dtype=kv_cache_dtype,
block_size=16,
),
)
return resolve_obj_by_qualname(path)
boundary_backend = resolve("auto")
quantized_backend = resolve("turboquant_k8v4")
assert quantized_backend is AttentionBackendEnum.TURBOQUANT.get_class()
# Raises when the intersection is empty, which is the startup failure.
assert get_supported_kv_cache_layouts([boundary_backend, quantized_backend])
def test_sparse_not_supported(mock_vllm_config):
"""Test that sparse MLA without use_mla flag raises an error."""
from vllm.platforms.rocm import RocmPlatform