[ROCm] Enable AITER and FP8 inference on GFX120x (#43615)

Signed-off-by: skysnow2001 <skysnow9285@gmail.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
skysnow2001
2026-08-03 16:45:37 -05:00
committed by GitHub
co-authored by Douglas Lehr
parent 4a3447d200
commit f43e1d26e3
12 changed files with 172 additions and 44 deletions
@@ -397,10 +397,14 @@ class AiterFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
@classmethod
def is_supported(cls, compute_capability=None):
if (
rocm_aiter_ops.is_linear_enabled()
or rocm_aiter_ops.is_rdna_linear_enabled()
):
return True, None
return (
rocm_aiter_ops.is_linear_enabled(),
"Only supported on ROCm platform \
with aiter package installed.",
False,
"Only supported on ROCm platform with aiter package installed.",
)
@classmethod
@@ -416,6 +420,17 @@ class AiterFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
"Supports only dynamic per token group activation "
"quantization with group_shape=(1,128).",
)
# RDNA4 (gfx12) only has the aiter Triton blockscale backend, which
# needs a per-(N,K) tune. Reject untuned shapes so the dispatcher falls
# through to the generic backend.
if rocm_aiter_ops.is_rdna_linear_enabled():
n, k = config.weight_shape
if not rocm_aiter_ops.is_triton_gemm_w8a8_tuned(n, k):
return (
False,
f"(N={n}, K={k}) is not in the aiter Triton blockscale tuned list.",
)
return True, None
def apply_block_scaled_mm(
@@ -108,10 +108,10 @@ class RowWiseTorchFP8ScaledMMLinearKernel(TorchFP8ScaledMMLinearKernel):
if not current_platform.is_rocm():
return False, "requires ROCm."
from vllm.platforms.rocm import get_cdna_version
from vllm.platforms.rocm import get_cdna_version, on_rdna4
if get_cdna_version() <= 2:
return False, "requires CDNA3+"
if get_cdna_version() <= 2 and not on_rdna4():
return False, "requires CDNA3+ or RDNA4"
if compute_capability is not None and compute_capability < 94:
return False, "requires compute capability 94 and above."
@@ -814,13 +814,13 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular):
) -> bool:
p = current_platform
if p.is_rocm():
from vllm.platforms.rocm import get_cdna_version
from vllm.platforms.rocm import get_cdna_version, on_rdna4
_rocm_support_fp8 = get_cdna_version() > 2
is_rocm_fp8_supported = get_cdna_version() > 2 or on_rdna4()
else:
_rocm_support_fp8 = False
is_rocm_fp8_supported = False
device_supports_fp8 = _rocm_support_fp8 or (
device_supports_fp8 = is_rocm_fp8_supported or (
p.is_cuda() and p.has_device_capability((8, 9))
)
@@ -379,8 +379,14 @@ def select_fp8_moe_backend(
# Handle explicit AITER FP8 configuration.
if envs.is_set("VLLM_ROCM_USE_AITER") or envs.is_set("VLLM_ROCM_USE_AITER_MOE"):
if not envs.VLLM_ROCM_USE_AITER or not envs.VLLM_ROCM_USE_AITER_MOE:
AVAILABLE_BACKENDS.remove(Fp8MoeBackend.AITER)
skip_aiter_moe = (
not envs.VLLM_ROCM_USE_AITER
or not envs.VLLM_ROCM_USE_AITER_MOE
or rocm_aiter_ops.is_rdna_aiter_enabled()
)
if skip_aiter_moe:
if Fp8MoeBackend.AITER in AVAILABLE_BACKENDS:
AVAILABLE_BACKENDS.remove(Fp8MoeBackend.AITER)
else:
backend = Fp8MoeBackend.AITER
return _return_or_raise(
@@ -298,7 +298,12 @@ def select_unquantized_moe_backend(
# Handle explicit AITER FP8 configuration.
if envs.is_set("VLLM_ROCM_USE_AITER") or envs.is_set("VLLM_ROCM_USE_AITER_MOE"):
if not envs.VLLM_ROCM_USE_AITER or not envs.VLLM_ROCM_USE_AITER_MOE:
skip_aiter_moe = (
not envs.VLLM_ROCM_USE_AITER
or not envs.VLLM_ROCM_USE_AITER_MOE
or rocm_aiter_ops.is_rdna_aiter_enabled()
)
if skip_aiter_moe:
if UnquantizedMoeBackend.AITER in AVAILABLE_BACKENDS:
AVAILABLE_BACKENDS.remove(UnquantizedMoeBackend.AITER)
else:
@@ -69,7 +69,10 @@ from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
# Availability is checked centrally via rocm_aiter_ops; the actual function
# references are imported here so that they can be called without per-call
# import overhead.
GDN_AITER_TRITON_AVAILABLE = rocm_aiter_ops.are_gdn_triton_kernels_available()
GDN_AITER_TRITON_AVAILABLE = (
rocm_aiter_ops.are_gdn_triton_kernels_available()
or rocm_aiter_ops.is_rdna_gdn_triton_kernels_available()
)
if GDN_AITER_TRITON_AVAILABLE:
from aiter.ops.triton.causal_conv1d_update_single_token import (
@@ -850,7 +850,7 @@ class SparseAttnIndexer(CustomOp):
assert isinstance(q_quant, torch.Tensor), (
"AMD sparse_attn_indexer expects a single FP8 q_quant tensor"
)
if rocm_aiter_ops.is_enabled():
if rocm_aiter_ops.is_enabled() or rocm_aiter_ops.is_rdna_aiter_enabled():
return torch.ops.vllm.rocm_aiter_sparse_attn_indexer(
hidden_states,
_encode_layer_name(self.k_cache.prefix),