[ROCm] Enable AITER and FP8 inference on GFX120x (#43615)

Signed-off-by: skysnow2001 <skysnow9285@gmail.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
skysnow2001
2026-08-03 16:45:37 -05:00
committed by GitHub
co-authored by Douglas Lehr
parent 4a3447d200
commit f43e1d26e3
12 changed files with 172 additions and 44 deletions
@@ -814,13 +814,13 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular):
) -> bool:
p = current_platform
if p.is_rocm():
from vllm.platforms.rocm import get_cdna_version
from vllm.platforms.rocm import get_cdna_version, on_rdna4
_rocm_support_fp8 = get_cdna_version() > 2
is_rocm_fp8_supported = get_cdna_version() > 2 or on_rdna4()
else:
_rocm_support_fp8 = False
is_rocm_fp8_supported = False
device_supports_fp8 = _rocm_support_fp8 or (
device_supports_fp8 = is_rocm_fp8_supported or (
p.is_cuda() and p.has_device_capability((8, 9))
)
@@ -379,8 +379,14 @@ def select_fp8_moe_backend(
# Handle explicit AITER FP8 configuration.
if envs.is_set("VLLM_ROCM_USE_AITER") or envs.is_set("VLLM_ROCM_USE_AITER_MOE"):
if not envs.VLLM_ROCM_USE_AITER or not envs.VLLM_ROCM_USE_AITER_MOE:
AVAILABLE_BACKENDS.remove(Fp8MoeBackend.AITER)
skip_aiter_moe = (
not envs.VLLM_ROCM_USE_AITER
or not envs.VLLM_ROCM_USE_AITER_MOE
or rocm_aiter_ops.is_rdna_aiter_enabled()
)
if skip_aiter_moe:
if Fp8MoeBackend.AITER in AVAILABLE_BACKENDS:
AVAILABLE_BACKENDS.remove(Fp8MoeBackend.AITER)
else:
backend = Fp8MoeBackend.AITER
return _return_or_raise(
@@ -298,7 +298,12 @@ def select_unquantized_moe_backend(
# Handle explicit AITER FP8 configuration.
if envs.is_set("VLLM_ROCM_USE_AITER") or envs.is_set("VLLM_ROCM_USE_AITER_MOE"):
if not envs.VLLM_ROCM_USE_AITER or not envs.VLLM_ROCM_USE_AITER_MOE:
skip_aiter_moe = (
not envs.VLLM_ROCM_USE_AITER
or not envs.VLLM_ROCM_USE_AITER_MOE
or rocm_aiter_ops.is_rdna_aiter_enabled()
)
if skip_aiter_moe:
if UnquantizedMoeBackend.AITER in AVAILABLE_BACKENDS:
AVAILABLE_BACKENDS.remove(UnquantizedMoeBackend.AITER)
else:
@@ -69,7 +69,10 @@ from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
# Availability is checked centrally via rocm_aiter_ops; the actual function
# references are imported here so that they can be called without per-call
# import overhead.
GDN_AITER_TRITON_AVAILABLE = rocm_aiter_ops.are_gdn_triton_kernels_available()
GDN_AITER_TRITON_AVAILABLE = (
rocm_aiter_ops.are_gdn_triton_kernels_available()
or rocm_aiter_ops.is_rdna_gdn_triton_kernels_available()
)
if GDN_AITER_TRITON_AVAILABLE:
from aiter.ops.triton.causal_conv1d_update_single_token import (
@@ -850,7 +850,7 @@ class SparseAttnIndexer(CustomOp):
assert isinstance(q_quant, torch.Tensor), (
"AMD sparse_attn_indexer expects a single FP8 q_quant tensor"
)
if rocm_aiter_ops.is_enabled():
if rocm_aiter_ops.is_enabled() or rocm_aiter_ops.is_rdna_aiter_enabled():
return torch.ops.vllm.rocm_aiter_sparse_attn_indexer(
hidden_states,
_encode_layer_name(self.k_cache.prefix),