[ROCm] Give turboquant boundary layers a layout-compatible backend (#54988)

Signed-off-by: Stefan Koncarevic <stefan.koncarevic@amd.com>
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
stefankoncarevic
2026-09-23 22:00:07 -05:00
committed by GitHub
co-authored by Andreas Karatzas Codex
parent e6dc16cebd
commit e78e367c6e
3 changed files with 232 additions and 16 deletions
+82 -16
View File
@@ -23,10 +23,14 @@ if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.config.kernel import IrOpPriorityConfig
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.v1.attention.backend import AttentionBackend
from vllm.v1.attention.selector import AttentionSelectorConfig
logger = init_logger(__name__)
_KV_CACHE_DTYPE_REASON = "kv_cache_dtype not supported"
_TURBOQUANT_LAYOUT_REASON = "no KV cache layout in common with TURBOQUANT"
try:
from amdsmi import (
AmdSmiException,
@@ -505,6 +509,53 @@ def _get_backend_priorities(
return backends
def _uses_turboquant(vllm_config: "VllmConfig | None") -> bool:
"""Whether the run's KV cache dtype is one of the turboquant_* presets."""
cache_config = getattr(vllm_config, "cache_config", None)
return cache_config is not None and str(cache_config.cache_dtype).startswith(
"turboquant_"
)
def _shares_layout_with_turboquant(backend_class: type["AttentionBackend"]) -> bool:
"""Whether this backend reads a KV cache layout TURBOQUANT reads too.
A turboquant_* run keeps its boundary layers at the native dtype (see
TurboQuantConfig.get_boundary_skip_layers), so those layers pick a backend of
their own while every other layer picks TURBOQUANT. One layout has to serve
the whole worker, and get_supported_kv_cache_layouts() turns an empty
intersection into a hard error at engine startup.
"""
layouts = backend_class.supported_kv_cache_layouts()
turboquant_layouts = (
AttentionBackendEnum.TURBOQUANT.get_class().supported_kv_cache_layouts()
)
if layouts is None or turboquant_layouts is None:
return True
return not set(layouts).isdisjoint(turboquant_layouts)
def _get_invalid_reasons(
backend_class: type["AttentionBackend"],
device_capability: DeviceCapability,
attn_selector_config: "AttentionSelectorConfig",
*,
is_turboquant_run: bool,
) -> list[str]:
"""Why this backend cannot serve the layer, empty when it can."""
invalid_reasons = backend_class.validate_configuration(
device_capability=device_capability,
**attn_selector_config._asdict(),
)
if (
not invalid_reasons
and is_turboquant_run
and not _shares_layout_with_turboquant(backend_class)
):
invalid_reasons = [_TURBOQUANT_LAYOUT_REASON]
return invalid_reasons
class RocmPlatform(Platform):
_enum = PlatformEnum.ROCM
device_name: str = "rocm"
@@ -611,12 +662,14 @@ class RocmPlatform(Platform):
# TODO: Make this explicit in the selector in a future PR.
if is_encoder_decoder and AttentionBackendEnum.ROCM_ATTN in backend_priorities:
backend_priorities.remove(AttentionBackendEnum.ROCM_ATTN)
is_turboquant_run = _uses_turboquant(vllm_config)
for priority, backend in enumerate(backend_priorities):
try:
backend_class = backend.get_class()
invalid_reasons_i = backend_class.validate_configuration(
device_capability=device_capability,
**attn_selector_config._asdict(),
invalid_reasons_i = _get_invalid_reasons(
backend.get_class(),
device_capability,
attn_selector_config,
is_turboquant_run=is_turboquant_run,
)
except ImportError:
invalid_reasons_i = ["ImportError"]
@@ -639,11 +692,16 @@ class RocmPlatform(Platform):
# First try checking just the selected backend, if there is one.
if selected_backend is not None:
# Keep lazy: vllm.config imports current_platform during initialization.
from vllm.config import get_current_vllm_config_or_none
is_turboquant_run = _uses_turboquant(get_current_vllm_config_or_none())
try:
backend_class = selected_backend.get_class()
sel_invalid_reasons = backend_class.validate_configuration(
device_capability=device_capability,
**attn_selector_config._asdict(),
sel_invalid_reasons = _get_invalid_reasons(
selected_backend.get_class(),
device_capability,
attn_selector_config,
is_turboquant_run=is_turboquant_run,
)
except ImportError:
sel_invalid_reasons = ["ImportError"]
@@ -653,21 +711,29 @@ class RocmPlatform(Platform):
selected_backend.name,
)
return selected_backend.get_path()
# Only tolerate the mismatch for turboquant_* KV-cache layers:
# boundary layers keep the native dtype (served by the selected
# backend) while turboquant_* layers need TURBOQUANT, so no single
# --attention-backend can serve every layer. For any other dtype
# the explicit selection is genuinely invalid -> fail loud.
# Only tolerate the mismatch when turboquant is in play: boundary
# layers keep the native dtype while every other layer needs
# TURBOQUANT, so no single --attention-backend can serve every layer.
# For any other dtype the selection is genuinely invalid -> fail loud.
kv_dtype = attn_selector_config.kv_cache_dtype
if not (kv_dtype is not None and str(kv_dtype).startswith("turboquant")):
layer_is_turboquant = kv_dtype is not None and str(kv_dtype).startswith(
"turboquant"
)
is_turboquant_fallback = (
is_turboquant_run or layer_is_turboquant
) and sel_invalid_reasons in (
[_KV_CACHE_DTYPE_REASON],
[_TURBOQUANT_LAYOUT_REASON],
)
if not is_turboquant_fallback:
raise ValueError(
f"Selected backend {selected_backend} is not valid for "
f"this configuration. Reason: {sel_invalid_reasons}"
)
# NOTE: pass a str (not the list) -- info_once hashes its args.
logger.info_once(
"Selected backend %s is incompatible with this turboquant "
"layer (%s); using the auto-selected per-layer backend. "
"Selected backend %s is incompatible with this layer (%s) of "
"the turboquant run; using the auto-selected per-layer backend. "
"Reason: %s",
selected_backend.name,
attn_selector_config.attn_type,