[ROCm] Enable AITER and FP8 inference on GFX120x (#43615)

Signed-off-by: skysnow2001 <skysnow9285@gmail.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
skysnow2001
2026-08-03 16:45:37 -05:00
committed by GitHub
co-authored by Douglas Lehr
parent 4a3447d200
commit f43e1d26e3
12 changed files with 172 additions and 44 deletions
+84 -14
View File
@@ -72,6 +72,21 @@ def is_aiter_found_and_supported() -> bool:
return False
def is_aiter_found_and_supported_on_rdna4() -> bool:
"""RDNA4 (gfx12) analog of `is_aiter_found_and_supported()`.
gfx12 has no aiter CK build, so this deliberately stays off the gfx9
`@if_aiter_supported` umbrella; it reports only that aiter's Triton
kernels are usable here. Like its gfx9 counterpart it checks platform +
arch + library availability and does not check environment variables.
"""
if current_platform.is_rocm() and IS_AITER_FOUND:
from vllm.platforms.rocm import on_rdna4
return on_rdna4()
return False
@functools.cache
def _load_gemm_tuned_configs(
q_dtype_w: torch.dtype, csv_path: str
@@ -1702,6 +1717,24 @@ class rocm_aiter_ops:
def is_enabled(cls) -> bool:
return cls._AITER_ENABLED
@classmethod
def is_rdna_aiter_enabled(cls) -> bool:
"""AITER on RDNA4 (gfx12): library present, arch is rdna4, and the user
enabled aiter. Only aiter's Triton kernels exist on gfx12 (no CK build),
so this deliberately stays off the gfx9/CK `@if_aiter_supported` umbrella
and gates only the Triton paths rdna4 uses. The gfx12 analog of
`is_enabled()`."""
if not current_platform.is_rocm() or not IS_AITER_FOUND:
return False
from vllm.platforms.rocm import on_rdna4
return on_rdna4() and cls._AITER_ENABLED
@classmethod
def is_rdna_linear_enabled(cls) -> bool:
"""RDNA4 (gfx12) analog of is_linear_enabled() (aiter Triton blockscale)."""
return cls.is_rdna_aiter_enabled() and cls._LINEAR_ENABLED
@classmethod
@if_aiter_supported
def is_linear_enabled(cls) -> bool:
@@ -1864,17 +1897,8 @@ class rocm_aiter_ops:
aiter_ar_comm if isinstance(aiter_ar_comm, AiterCustomAllreduce) else None
)
@classmethod
@if_aiter_supported
def are_gdn_triton_kernels_available(cls) -> bool:
"""Check if AITER Triton kernels for GDN attention are importable.
These are optional Triton kernels (conv1d fast-path, gated delta net)
used by GatedDeltaNetAttention's decode fast-path. They may be absent
in older aiter builds.
"""
if not cls._AITER_ENABLED:
return False
@staticmethod
def _gdn_triton_kernels_importable() -> bool:
try:
import aiter.ops.triton.causal_conv1d_update_single_token # noqa: F401
import aiter.ops.triton.gated_delta_net # noqa: F401
@@ -1886,6 +1910,22 @@ class rocm_aiter_ops:
except (ImportError, ModuleNotFoundError):
return False
@classmethod
@if_aiter_supported
def are_gdn_triton_kernels_available(cls) -> bool:
"""Check if AITER Triton kernels for GDN attention are importable.
These are optional Triton kernels (conv1d fast-path, gated delta net)
used by GatedDeltaNetAttention's decode fast-path. They may be absent
in older aiter builds.
"""
return cls._AITER_ENABLED and cls._gdn_triton_kernels_importable()
@classmethod
def is_rdna_gdn_triton_kernels_available(cls) -> bool:
"""RDNA4 (gfx12) analog of are_gdn_triton_kernels_available()."""
return cls.is_rdna_aiter_enabled() and cls._gdn_triton_kernels_importable()
@classmethod
@if_aiter_supported
@functools.cache
@@ -1902,8 +1942,12 @@ class rocm_aiter_ops:
return "gate_mode" in inspect.signature(fused_moe).parameters
@staticmethod
@if_aiter_supported
def register_ops_once() -> None:
if not (
is_aiter_found_and_supported() or is_aiter_found_and_supported_on_rdna4()
):
return
global _OPS_REGISTERED
if not _OPS_REGISTERED:
# register all the custom ops here
@@ -2774,7 +2818,11 @@ class rocm_aiter_ops:
@staticmethod
def is_triton_gemm_w8a8_tuned(n: int, k: int) -> bool:
return (n, k) in [
if not current_platform.is_rocm():
return False
from vllm.platforms.rocm import on_gfx950, on_rdna4
gfx950_tuned = {
(1024, 8192),
(2112, 7168),
(3072, 1536),
@@ -2786,7 +2834,29 @@ class rocm_aiter_ops:
(7168, 256),
(8192, 1024),
(8192, 32768),
]
}
rdna4_tuned = gfx950_tuned | {
(2048, 2048),
(2624, 6144),
(3072, 6144),
(3584, 512),
(4096, 512),
(6144, 1536),
(6144, 2048),
(7168, 2304),
(7168, 16384),
(7168, 18432),
(8192, 8192),
(16384, 1536),
(24576, 1536),
(32768, 512),
(36864, 7168),
}
if on_rdna4():
return (n, k) in rdna4_tuned
if on_gfx950():
return (n, k) in gfx950_tuned
return False
@staticmethod
def is_triton_gemm_afp4wfp4_presh_ws_tuned(n: int, k: int) -> bool:
@@ -581,6 +581,9 @@ class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
gated_norm_shapes.add((num_v_heads // layer.tp_size, head_v_dim))
# RDNA4 uses native quant ops and supports only Triton replacements.
match_aiter_quant_op = not rocm_aiter_ops.is_rdna_aiter_enabled()
# Make sure fused add patterns are before simple rms norm,
# as the latter is a subset of the former in torch ops.
# The DoubleQuant patterns handle 1 rms_norm -> 2 group_fp8_quant
@@ -592,22 +595,22 @@ class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
for epsilon in [1e-5, 1e-6]:
# Fuse aiter rms_norm + 2x aiter group fp8 quant
DoubleAiterRMSFp8GroupQuantPattern(
epsilon, FP8_DTYPE, GroupShape(1, 128)
epsilon, FP8_DTYPE, GroupShape(1, 128), match_aiter_quant_op
).register(self.patterns)
# View-tolerant sibling for DSv3.2 q_c norm fan-out
DoubleAiterRMSFp8GroupQuantViewPattern(
epsilon, FP8_DTYPE, GroupShape(1, 128)
epsilon, FP8_DTYPE, GroupShape(1, 128), match_aiter_quant_op
).register(self.patterns)
# Fuse aiter rms_norm + aiter dynamic group fp8 quant
AiterRMSFp8GroupQuantPattern(
epsilon, FP8_DTYPE, GroupShape(1, 128)
epsilon, FP8_DTYPE, GroupShape(1, 128), match_aiter_quant_op
).register(self.patterns)
# Fuse aiter fused_add_rms_norm + aiter dynamic group fp8 quant
AiterFusedAddRMSFp8GroupQuantPattern(
epsilon, FP8_DTYPE, GroupShape(1, 128)
epsilon, FP8_DTYPE, GroupShape(1, 128), match_aiter_quant_op
).register(self.patterns)
# When quant_fp8 custom ops are disabled, both AITER and native
@@ -620,6 +623,8 @@ class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
match_aiter_quant_options = (
[True, False] if is_quant_fp8_enabled else [False]
)
if not match_aiter_quant_op:
match_aiter_quant_options = []
for match_aiter_quant in match_aiter_quant_options:
# Fuse aiter rms_norm + (aiter / vllm built-in)
@@ -637,7 +642,10 @@ class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
# Fuse decomposed RMSNormGated + group fp8 quant.
# The replacement op (fused_rms_gated_fp8_group_quant) requires
# an aiter version that includes the GDN triton kernel renames.
if gated_norm_shapes and rocm_aiter_ops.are_gdn_triton_kernels_available():
if gated_norm_shapes and (
rocm_aiter_ops.are_gdn_triton_kernels_available()
or rocm_aiter_ops.is_rdna_gdn_triton_kernels_available()
):
for num_heads, head_dim in gated_norm_shapes:
if head_dim != 128:
continue
@@ -647,6 +655,7 @@ class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
GroupShape(1, 128),
num_heads=num_heads,
head_dim=head_dim,
match_aiter_quant=match_aiter_quant_op,
).register(self.patterns)
self.dump_patterns(config, self.patterns)
@@ -679,10 +688,10 @@ class AiterSiluMulFp8GroupQuantPattern(VllmPatternReplacement):
FUSED_SILU_MUL_QUANT_OP = rocm_aiter_ops.get_act_mul_fused_fp8_group_quant_op()
def __init__(self) -> None:
def __init__(self, match_aiter_quant_op: bool = True) -> None:
self.silu_and_mul_matcher = MatcherSiluAndMul()
self.quant_matcher = MatcherQuantFP8(
quant_key=kFp8Dynamic128Sym, match_rocm_aiter=True
quant_key=kFp8Dynamic128Sym, match_rocm_aiter=match_aiter_quant_op
)
def get_inputs(self) -> list[torch.Tensor]:
@@ -725,7 +734,11 @@ class RocmAiterSiluMulFp8GroupQuantFusionPass(VllmFusionPatternMatcherPass):
def __init__(self, config: VllmConfig) -> None:
super().__init__(config, "rocm_aiter_silu_mul_fp8_group_quant_fusion_pass")
self.register(AiterSiluMulFp8GroupQuantPattern())
match_aiter_quant_op = not rocm_aiter_ops.is_rdna_aiter_enabled()
self.register(
AiterSiluMulFp8GroupQuantPattern(match_aiter_quant_op=match_aiter_quant_op)
)
self.dump_patterns(config, self.pm_pass)
+12 -4
View File
@@ -18,7 +18,7 @@ from .ir.clone_elimination import UnsafeCloneEliminationPass
from .ir.lowering_pass import VllmIRLoweringPass
from .vllm_inductor_pass import VllmInductorPass, VllmPatternMatcherPass
if rocm_aiter_ops.is_enabled():
if rocm_aiter_ops.is_enabled() or rocm_aiter_ops.is_rdna_aiter_enabled():
from .fusion.allreduce_rms_fusion import (
RocmAiterAllReduceFusionPass,
)
@@ -166,7 +166,9 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
if enable_transformers_norm_canonicalization:
self.passes += [AddRMSNormFusionPass(config)]
if self.pass_config.fuse_act_padding and rocm_aiter_ops.is_enabled():
if self.pass_config.fuse_act_padding and (
rocm_aiter_ops.is_enabled() or rocm_aiter_ops.is_rdna_aiter_enabled()
):
# Run the more specific RMSNorm+router-pad fusion before
# AR+RMS, since both consume fused_add_rms_norm.
self.passes += [RocmAiterTritonAddRMSNormPadFusionPass(config)]
@@ -183,7 +185,10 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
self.passes += [RMSNormReshapeFusionPass(config)]
if self.pass_config.fuse_norm_quant:
if rocm_aiter_ops.is_enabled():
if (
rocm_aiter_ops.is_enabled()
or rocm_aiter_ops.is_rdna_aiter_enabled()
):
self.passes += [
RocmAiterRMSNormQuantFusionPass(config),
]
@@ -191,7 +196,10 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
if self.pass_config.fuse_act_quant:
self.passes += [ActivationQuantFusionPass(config)]
if rocm_aiter_ops.is_enabled():
if (
rocm_aiter_ops.is_enabled()
or rocm_aiter_ops.is_rdna_aiter_enabled()
):
self.passes += [RocmAiterSiluMulFp8GroupQuantFusionPass(config)]
if self.pass_config.fuse_qk_norm_rope_kvcache:
@@ -397,10 +397,14 @@ class AiterFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
@classmethod
def is_supported(cls, compute_capability=None):
if (
rocm_aiter_ops.is_linear_enabled()
or rocm_aiter_ops.is_rdna_linear_enabled()
):
return True, None
return (
rocm_aiter_ops.is_linear_enabled(),
"Only supported on ROCm platform \
with aiter package installed.",
False,
"Only supported on ROCm platform with aiter package installed.",
)
@classmethod
@@ -416,6 +420,17 @@ class AiterFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
"Supports only dynamic per token group activation "
"quantization with group_shape=(1,128).",
)
# RDNA4 (gfx12) only has the aiter Triton blockscale backend, which
# needs a per-(N,K) tune. Reject untuned shapes so the dispatcher falls
# through to the generic backend.
if rocm_aiter_ops.is_rdna_linear_enabled():
n, k = config.weight_shape
if not rocm_aiter_ops.is_triton_gemm_w8a8_tuned(n, k):
return (
False,
f"(N={n}, K={k}) is not in the aiter Triton blockscale tuned list.",
)
return True, None
def apply_block_scaled_mm(
@@ -108,10 +108,10 @@ class RowWiseTorchFP8ScaledMMLinearKernel(TorchFP8ScaledMMLinearKernel):
if not current_platform.is_rocm():
return False, "requires ROCm."
from vllm.platforms.rocm import get_cdna_version
from vllm.platforms.rocm import get_cdna_version, on_rdna4
if get_cdna_version() <= 2:
return False, "requires CDNA3+"
if get_cdna_version() <= 2 and not on_rdna4():
return False, "requires CDNA3+ or RDNA4"
if compute_capability is not None and compute_capability < 94:
return False, "requires compute capability 94 and above."
@@ -814,13 +814,13 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular):
) -> bool:
p = current_platform
if p.is_rocm():
from vllm.platforms.rocm import get_cdna_version
from vllm.platforms.rocm import get_cdna_version, on_rdna4
_rocm_support_fp8 = get_cdna_version() > 2
is_rocm_fp8_supported = get_cdna_version() > 2 or on_rdna4()
else:
_rocm_support_fp8 = False
is_rocm_fp8_supported = False
device_supports_fp8 = _rocm_support_fp8 or (
device_supports_fp8 = is_rocm_fp8_supported or (
p.is_cuda() and p.has_device_capability((8, 9))
)
@@ -379,8 +379,14 @@ def select_fp8_moe_backend(
# Handle explicit AITER FP8 configuration.
if envs.is_set("VLLM_ROCM_USE_AITER") or envs.is_set("VLLM_ROCM_USE_AITER_MOE"):
if not envs.VLLM_ROCM_USE_AITER or not envs.VLLM_ROCM_USE_AITER_MOE:
AVAILABLE_BACKENDS.remove(Fp8MoeBackend.AITER)
skip_aiter_moe = (
not envs.VLLM_ROCM_USE_AITER
or not envs.VLLM_ROCM_USE_AITER_MOE
or rocm_aiter_ops.is_rdna_aiter_enabled()
)
if skip_aiter_moe:
if Fp8MoeBackend.AITER in AVAILABLE_BACKENDS:
AVAILABLE_BACKENDS.remove(Fp8MoeBackend.AITER)
else:
backend = Fp8MoeBackend.AITER
return _return_or_raise(
@@ -298,7 +298,12 @@ def select_unquantized_moe_backend(
# Handle explicit AITER FP8 configuration.
if envs.is_set("VLLM_ROCM_USE_AITER") or envs.is_set("VLLM_ROCM_USE_AITER_MOE"):
if not envs.VLLM_ROCM_USE_AITER or not envs.VLLM_ROCM_USE_AITER_MOE:
skip_aiter_moe = (
not envs.VLLM_ROCM_USE_AITER
or not envs.VLLM_ROCM_USE_AITER_MOE
or rocm_aiter_ops.is_rdna_aiter_enabled()
)
if skip_aiter_moe:
if UnquantizedMoeBackend.AITER in AVAILABLE_BACKENDS:
AVAILABLE_BACKENDS.remove(UnquantizedMoeBackend.AITER)
else:
@@ -69,7 +69,10 @@ from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
# Availability is checked centrally via rocm_aiter_ops; the actual function
# references are imported here so that they can be called without per-call
# import overhead.
GDN_AITER_TRITON_AVAILABLE = rocm_aiter_ops.are_gdn_triton_kernels_available()
GDN_AITER_TRITON_AVAILABLE = (
rocm_aiter_ops.are_gdn_triton_kernels_available()
or rocm_aiter_ops.is_rdna_gdn_triton_kernels_available()
)
if GDN_AITER_TRITON_AVAILABLE:
from aiter.ops.triton.causal_conv1d_update_single_token import (
@@ -850,7 +850,7 @@ class SparseAttnIndexer(CustomOp):
assert isinstance(q_quant, torch.Tensor), (
"AMD sparse_attn_indexer expects a single FP8 q_quant tensor"
)
if rocm_aiter_ops.is_enabled():
if rocm_aiter_ops.is_enabled() or rocm_aiter_ops.is_rdna_aiter_enabled():
return torch.ops.vllm.rocm_aiter_sparse_attn_indexer(
hidden_states,
_encode_layer_name(self.k_cache.prefix),
+9 -1
View File
@@ -218,6 +218,7 @@ _ON_GFX1250 = "gfx1250" in _GCN_ARCH
_ON_CDNA = any(arch in _GCN_ARCH for arch in ["gfx9", "gfx1250"])
# RDNA = gfx11/gfx12 minus the CDNA-classified gfx1250.
_ON_RDNA = _ON_GFX1X and not _ON_CDNA
_ON_RDNA4 = any(arch in _GCN_ARCH for arch in ["gfx1200", "gfx1201"])
def _capability_from_gcn_arch(gcn_arch: str) -> tuple[int, int] | None:
@@ -315,6 +316,10 @@ def on_gfx1250() -> bool:
return _ON_GFX1250
def on_rdna4() -> bool:
return _ON_RDNA4
def on_mi3xx() -> bool:
return _ON_MI3XX
@@ -471,6 +476,8 @@ def _get_backend_priorities(
backends.append(AttentionBackendEnum.ROCM_AITER_FA)
if is_aiter_found_and_supported():
backends.append(AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN)
elif rocm_aiter_ops.is_rdna_aiter_enabled():
backends.insert(0, AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN)
backends.append(AttentionBackendEnum.TRITON_ATTN)
backends.append(AttentionBackendEnum.TURBOQUANT)
@@ -918,7 +925,7 @@ class RocmPlatform(Platform):
@classmethod
def supports_fp8(cls) -> bool:
return on_cdna() or on_gfx12x()
return on_cdna() or on_rdna4()
@classmethod
def is_fp8_fnuz(cls) -> bool:
@@ -1065,6 +1072,7 @@ class RocmPlatform(Platform):
cc.cudagraph_mode != CUDAGraphMode.NONE
and envs.VLLM_ROCM_USE_AITER
and envs.VLLM_ROCM_USE_AITER_RMSNORM
and not on_rdna4()
):
rms_norm = ["aiter"] + default
else:
@@ -417,7 +417,7 @@ def rocm_fp8_paged_mqa_logits(
batch_size, next_n = q_fp8.shape[:2]
block_size = kv_cache_fp8.shape[1]
if rocm_aiter_ops.is_enabled():
if rocm_aiter_ops.is_enabled() or rocm_aiter_ops.is_rdna_aiter_enabled():
aiter_paged_mqa_logits_module = paged_mqa_logits_module()
if aiter_paged_mqa_logits_module is not None:
@@ -582,7 +582,7 @@ def rocm_fp8_mqa_logits(
)
aiter_mqa_logits_module = None
if rocm_aiter_ops.is_enabled():
if rocm_aiter_ops.is_enabled() or rocm_aiter_ops.is_rdna_aiter_enabled():
aiter_mqa_logits_module = mqa_logits_module()
if aiter_mqa_logits_module is not None: