[ROCm] Give turboquant boundary layers a layout-compatible backend (#54988)
Signed-off-by: Stefan Koncarevic <stefan.koncarevic@amd.com> Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Andreas Karatzas
Codex
parent
e6dc16cebd
commit
e78e367c6e
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for attention backend selectors."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
@@ -34,6 +35,15 @@ def mock_get_cdna_version():
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def cleared_attention_selector_cache():
|
||||
from vllm.v1.attention.selector import _cached_get_attn_backend
|
||||
|
||||
_cached_get_attn_backend.cache_clear()
|
||||
yield
|
||||
_cached_get_attn_backend.cache_clear()
|
||||
|
||||
|
||||
def test_aiter_unified_attention_uses_dedicated_metadata_builder():
|
||||
from vllm.v1.attention.backends.rocm_aiter_unified_attn import (
|
||||
RocmAiterUnifiedAttentionBackend,
|
||||
@@ -434,6 +444,139 @@ def test_aiter_fa_requires_mi3xx(mock_vllm_config):
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def turboquant_run_config():
|
||||
"""Current config of a run whose KV cache dtype is a turboquant_* preset."""
|
||||
config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(cache_dtype="turboquant_k8v4")
|
||||
)
|
||||
with patch(
|
||||
"vllm.config.get_current_vllm_config_or_none",
|
||||
return_value=config,
|
||||
):
|
||||
yield config
|
||||
|
||||
|
||||
def test_turboquant_boundary_selection_is_not_cached_from_ordinary_run(
|
||||
cleared_attention_selector_cache,
|
||||
):
|
||||
from vllm.config import CacheConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.v1.attention.backends.utils import get_supported_kv_cache_layouts
|
||||
from vllm.v1.attention.selector import get_attn_backend
|
||||
from vllm.v1.kv_cache_layout import KVCacheLayout
|
||||
|
||||
ordinary_config = VllmConfig(cache_config=CacheConfig(cache_dtype="auto"))
|
||||
turboquant_config = VllmConfig(
|
||||
cache_config=CacheConfig(cache_dtype="turboquant_k8v4")
|
||||
)
|
||||
|
||||
backend_priorities = [
|
||||
AttentionBackendEnum.ROCM_ATTN,
|
||||
AttentionBackendEnum.TRITON_ATTN,
|
||||
AttentionBackendEnum.TURBOQUANT,
|
||||
]
|
||||
with patch(
|
||||
"vllm.platforms.rocm._get_backend_priorities",
|
||||
return_value=backend_priorities,
|
||||
):
|
||||
with set_current_vllm_config(ordinary_config):
|
||||
ordinary_backend = get_attn_backend(128, torch.float16, "auto")
|
||||
|
||||
with set_current_vllm_config(turboquant_config):
|
||||
boundary_backend = get_attn_backend(128, torch.float16, "auto")
|
||||
quantized_backend = get_attn_backend(128, torch.float16, "turboquant_k8v4")
|
||||
|
||||
assert ordinary_backend is AttentionBackendEnum.ROCM_ATTN.get_class()
|
||||
assert quantized_backend is AttentionBackendEnum.TURBOQUANT.get_class()
|
||||
assert get_supported_kv_cache_layouts([boundary_backend, quantized_backend]) == [
|
||||
KVCacheLayout.LBNHC
|
||||
]
|
||||
|
||||
|
||||
def test_turboquant_run_does_not_mask_unsupported_selected_backend(
|
||||
turboquant_run_config,
|
||||
):
|
||||
from vllm.platforms.rocm import RocmPlatform
|
||||
|
||||
attn_selector_config = AttentionSelectorConfig(
|
||||
head_size=128,
|
||||
dtype=torch.float16,
|
||||
kv_cache_dtype="auto",
|
||||
block_size=16,
|
||||
)
|
||||
|
||||
with (
|
||||
patch("vllm.platforms.rocm.get_cdna_version", return_value=1),
|
||||
pytest.raises(ValueError, match="compute capability not supported"),
|
||||
):
|
||||
RocmPlatform.get_attn_backend_cls(
|
||||
selected_backend=AttentionBackendEnum.ROCM_AITER_FA,
|
||||
attn_selector_config=attn_selector_config,
|
||||
)
|
||||
|
||||
|
||||
def test_turboquant_layout_check_respects_backend_override():
|
||||
from vllm.platforms.rocm import _shares_layout_with_turboquant
|
||||
from vllm.v1.kv_cache_layout import KVCacheLayout
|
||||
|
||||
boundary_backend = MagicMock()
|
||||
boundary_backend.supported_kv_cache_layouts.return_value = (KVCacheLayout.LBHNC,)
|
||||
turboquant_override = MagicMock()
|
||||
turboquant_override.supported_kv_cache_layouts.return_value = (KVCacheLayout.LBHNC,)
|
||||
|
||||
with patch.object(
|
||||
AttentionBackendEnum.TURBOQUANT,
|
||||
"get_class",
|
||||
return_value=turboquant_override,
|
||||
) as get_turboquant_class:
|
||||
assert _shares_layout_with_turboquant(boundary_backend)
|
||||
get_turboquant_class.assert_called_once_with()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"selected_backend",
|
||||
[None, "ROCM_ATTN", "ROCM_AITER_FA", "TURBOQUANT"],
|
||||
)
|
||||
def test_turboquant_boundary_layers_share_a_layout(
|
||||
selected_backend,
|
||||
turboquant_run_config,
|
||||
mock_get_cdna_version,
|
||||
):
|
||||
"""A turboquant_* run must resolve to backends with a common KV layout.
|
||||
|
||||
The boundary layers keep the native dtype and pick their own backend while
|
||||
every other layer picks TURBOQUANT. Engine startup hard-errors when the two
|
||||
share no layout, so the boundary layers must not land on a native ROCm or
|
||||
AITER backend, whichever backend the run asked for.
|
||||
"""
|
||||
from vllm.platforms.rocm import RocmPlatform
|
||||
from vllm.utils.import_utils import resolve_obj_by_qualname
|
||||
from vllm.v1.attention.backends.utils import get_supported_kv_cache_layouts
|
||||
|
||||
backend_enum = (
|
||||
getattr(AttentionBackendEnum, selected_backend) if selected_backend else None
|
||||
)
|
||||
|
||||
def resolve(kv_cache_dtype):
|
||||
path = RocmPlatform.get_attn_backend_cls(
|
||||
selected_backend=backend_enum,
|
||||
attn_selector_config=AttentionSelectorConfig(
|
||||
head_size=128,
|
||||
dtype=torch.float16,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
block_size=16,
|
||||
),
|
||||
)
|
||||
return resolve_obj_by_qualname(path)
|
||||
|
||||
boundary_backend = resolve("auto")
|
||||
quantized_backend = resolve("turboquant_k8v4")
|
||||
|
||||
assert quantized_backend is AttentionBackendEnum.TURBOQUANT.get_class()
|
||||
# Raises when the intersection is empty, which is the startup failure.
|
||||
assert get_supported_kv_cache_layouts([boundary_backend, quantized_backend])
|
||||
|
||||
|
||||
def test_sparse_not_supported(mock_vllm_config):
|
||||
"""Test that sparse MLA without use_mla flag raises an error."""
|
||||
from vllm.platforms.rocm import RocmPlatform
|
||||
|
||||
Reference in New Issue
Block a user