[ROCm] Enable AITER and FP8 inference on GFX120x (#43615)
Signed-off-by: skysnow2001 <skysnow9285@gmail.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
co-authored by
Douglas Lehr
parent
4a3447d200
commit
f43e1d26e3
+84
-14
@@ -72,6 +72,21 @@ def is_aiter_found_and_supported() -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def is_aiter_found_and_supported_on_rdna4() -> bool:
|
||||
"""RDNA4 (gfx12) analog of `is_aiter_found_and_supported()`.
|
||||
|
||||
gfx12 has no aiter CK build, so this deliberately stays off the gfx9
|
||||
`@if_aiter_supported` umbrella; it reports only that aiter's Triton
|
||||
kernels are usable here. Like its gfx9 counterpart it checks platform +
|
||||
arch + library availability and does not check environment variables.
|
||||
"""
|
||||
if current_platform.is_rocm() and IS_AITER_FOUND:
|
||||
from vllm.platforms.rocm import on_rdna4
|
||||
|
||||
return on_rdna4()
|
||||
return False
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _load_gemm_tuned_configs(
|
||||
q_dtype_w: torch.dtype, csv_path: str
|
||||
@@ -1702,6 +1717,24 @@ class rocm_aiter_ops:
|
||||
def is_enabled(cls) -> bool:
|
||||
return cls._AITER_ENABLED
|
||||
|
||||
@classmethod
|
||||
def is_rdna_aiter_enabled(cls) -> bool:
|
||||
"""AITER on RDNA4 (gfx12): library present, arch is rdna4, and the user
|
||||
enabled aiter. Only aiter's Triton kernels exist on gfx12 (no CK build),
|
||||
so this deliberately stays off the gfx9/CK `@if_aiter_supported` umbrella
|
||||
and gates only the Triton paths rdna4 uses. The gfx12 analog of
|
||||
`is_enabled()`."""
|
||||
if not current_platform.is_rocm() or not IS_AITER_FOUND:
|
||||
return False
|
||||
from vllm.platforms.rocm import on_rdna4
|
||||
|
||||
return on_rdna4() and cls._AITER_ENABLED
|
||||
|
||||
@classmethod
|
||||
def is_rdna_linear_enabled(cls) -> bool:
|
||||
"""RDNA4 (gfx12) analog of is_linear_enabled() (aiter Triton blockscale)."""
|
||||
return cls.is_rdna_aiter_enabled() and cls._LINEAR_ENABLED
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def is_linear_enabled(cls) -> bool:
|
||||
@@ -1864,17 +1897,8 @@ class rocm_aiter_ops:
|
||||
aiter_ar_comm if isinstance(aiter_ar_comm, AiterCustomAllreduce) else None
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def are_gdn_triton_kernels_available(cls) -> bool:
|
||||
"""Check if AITER Triton kernels for GDN attention are importable.
|
||||
|
||||
These are optional Triton kernels (conv1d fast-path, gated delta net)
|
||||
used by GatedDeltaNetAttention's decode fast-path. They may be absent
|
||||
in older aiter builds.
|
||||
"""
|
||||
if not cls._AITER_ENABLED:
|
||||
return False
|
||||
@staticmethod
|
||||
def _gdn_triton_kernels_importable() -> bool:
|
||||
try:
|
||||
import aiter.ops.triton.causal_conv1d_update_single_token # noqa: F401
|
||||
import aiter.ops.triton.gated_delta_net # noqa: F401
|
||||
@@ -1886,6 +1910,22 @@ class rocm_aiter_ops:
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def are_gdn_triton_kernels_available(cls) -> bool:
|
||||
"""Check if AITER Triton kernels for GDN attention are importable.
|
||||
|
||||
These are optional Triton kernels (conv1d fast-path, gated delta net)
|
||||
used by GatedDeltaNetAttention's decode fast-path. They may be absent
|
||||
in older aiter builds.
|
||||
"""
|
||||
return cls._AITER_ENABLED and cls._gdn_triton_kernels_importable()
|
||||
|
||||
@classmethod
|
||||
def is_rdna_gdn_triton_kernels_available(cls) -> bool:
|
||||
"""RDNA4 (gfx12) analog of are_gdn_triton_kernels_available()."""
|
||||
return cls.is_rdna_aiter_enabled() and cls._gdn_triton_kernels_importable()
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
@functools.cache
|
||||
@@ -1902,8 +1942,12 @@ class rocm_aiter_ops:
|
||||
return "gate_mode" in inspect.signature(fused_moe).parameters
|
||||
|
||||
@staticmethod
|
||||
@if_aiter_supported
|
||||
def register_ops_once() -> None:
|
||||
if not (
|
||||
is_aiter_found_and_supported() or is_aiter_found_and_supported_on_rdna4()
|
||||
):
|
||||
return
|
||||
|
||||
global _OPS_REGISTERED
|
||||
if not _OPS_REGISTERED:
|
||||
# register all the custom ops here
|
||||
@@ -2774,7 +2818,11 @@ class rocm_aiter_ops:
|
||||
|
||||
@staticmethod
|
||||
def is_triton_gemm_w8a8_tuned(n: int, k: int) -> bool:
|
||||
return (n, k) in [
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx950, on_rdna4
|
||||
|
||||
gfx950_tuned = {
|
||||
(1024, 8192),
|
||||
(2112, 7168),
|
||||
(3072, 1536),
|
||||
@@ -2786,7 +2834,29 @@ class rocm_aiter_ops:
|
||||
(7168, 256),
|
||||
(8192, 1024),
|
||||
(8192, 32768),
|
||||
]
|
||||
}
|
||||
rdna4_tuned = gfx950_tuned | {
|
||||
(2048, 2048),
|
||||
(2624, 6144),
|
||||
(3072, 6144),
|
||||
(3584, 512),
|
||||
(4096, 512),
|
||||
(6144, 1536),
|
||||
(6144, 2048),
|
||||
(7168, 2304),
|
||||
(7168, 16384),
|
||||
(7168, 18432),
|
||||
(8192, 8192),
|
||||
(16384, 1536),
|
||||
(24576, 1536),
|
||||
(32768, 512),
|
||||
(36864, 7168),
|
||||
}
|
||||
if on_rdna4():
|
||||
return (n, k) in rdna4_tuned
|
||||
if on_gfx950():
|
||||
return (n, k) in gfx950_tuned
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def is_triton_gemm_afp4wfp4_presh_ws_tuned(n: int, k: int) -> bool:
|
||||
|
||||
@@ -581,6 +581,9 @@ class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
|
||||
|
||||
gated_norm_shapes.add((num_v_heads // layer.tp_size, head_v_dim))
|
||||
|
||||
# RDNA4 uses native quant ops and supports only Triton replacements.
|
||||
match_aiter_quant_op = not rocm_aiter_ops.is_rdna_aiter_enabled()
|
||||
|
||||
# Make sure fused add patterns are before simple rms norm,
|
||||
# as the latter is a subset of the former in torch ops.
|
||||
# The DoubleQuant patterns handle 1 rms_norm -> 2 group_fp8_quant
|
||||
@@ -592,22 +595,22 @@ class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
|
||||
for epsilon in [1e-5, 1e-6]:
|
||||
# Fuse aiter rms_norm + 2x aiter group fp8 quant
|
||||
DoubleAiterRMSFp8GroupQuantPattern(
|
||||
epsilon, FP8_DTYPE, GroupShape(1, 128)
|
||||
epsilon, FP8_DTYPE, GroupShape(1, 128), match_aiter_quant_op
|
||||
).register(self.patterns)
|
||||
|
||||
# View-tolerant sibling for DSv3.2 q_c norm fan-out
|
||||
DoubleAiterRMSFp8GroupQuantViewPattern(
|
||||
epsilon, FP8_DTYPE, GroupShape(1, 128)
|
||||
epsilon, FP8_DTYPE, GroupShape(1, 128), match_aiter_quant_op
|
||||
).register(self.patterns)
|
||||
|
||||
# Fuse aiter rms_norm + aiter dynamic group fp8 quant
|
||||
AiterRMSFp8GroupQuantPattern(
|
||||
epsilon, FP8_DTYPE, GroupShape(1, 128)
|
||||
epsilon, FP8_DTYPE, GroupShape(1, 128), match_aiter_quant_op
|
||||
).register(self.patterns)
|
||||
|
||||
# Fuse aiter fused_add_rms_norm + aiter dynamic group fp8 quant
|
||||
AiterFusedAddRMSFp8GroupQuantPattern(
|
||||
epsilon, FP8_DTYPE, GroupShape(1, 128)
|
||||
epsilon, FP8_DTYPE, GroupShape(1, 128), match_aiter_quant_op
|
||||
).register(self.patterns)
|
||||
|
||||
# When quant_fp8 custom ops are disabled, both AITER and native
|
||||
@@ -620,6 +623,8 @@ class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
|
||||
match_aiter_quant_options = (
|
||||
[True, False] if is_quant_fp8_enabled else [False]
|
||||
)
|
||||
if not match_aiter_quant_op:
|
||||
match_aiter_quant_options = []
|
||||
|
||||
for match_aiter_quant in match_aiter_quant_options:
|
||||
# Fuse aiter rms_norm + (aiter / vllm built-in)
|
||||
@@ -637,7 +642,10 @@ class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
|
||||
# Fuse decomposed RMSNormGated + group fp8 quant.
|
||||
# The replacement op (fused_rms_gated_fp8_group_quant) requires
|
||||
# an aiter version that includes the GDN triton kernel renames.
|
||||
if gated_norm_shapes and rocm_aiter_ops.are_gdn_triton_kernels_available():
|
||||
if gated_norm_shapes and (
|
||||
rocm_aiter_ops.are_gdn_triton_kernels_available()
|
||||
or rocm_aiter_ops.is_rdna_gdn_triton_kernels_available()
|
||||
):
|
||||
for num_heads, head_dim in gated_norm_shapes:
|
||||
if head_dim != 128:
|
||||
continue
|
||||
@@ -647,6 +655,7 @@ class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
|
||||
GroupShape(1, 128),
|
||||
num_heads=num_heads,
|
||||
head_dim=head_dim,
|
||||
match_aiter_quant=match_aiter_quant_op,
|
||||
).register(self.patterns)
|
||||
|
||||
self.dump_patterns(config, self.patterns)
|
||||
@@ -679,10 +688,10 @@ class AiterSiluMulFp8GroupQuantPattern(VllmPatternReplacement):
|
||||
|
||||
FUSED_SILU_MUL_QUANT_OP = rocm_aiter_ops.get_act_mul_fused_fp8_group_quant_op()
|
||||
|
||||
def __init__(self) -> None:
|
||||
def __init__(self, match_aiter_quant_op: bool = True) -> None:
|
||||
self.silu_and_mul_matcher = MatcherSiluAndMul()
|
||||
self.quant_matcher = MatcherQuantFP8(
|
||||
quant_key=kFp8Dynamic128Sym, match_rocm_aiter=True
|
||||
quant_key=kFp8Dynamic128Sym, match_rocm_aiter=match_aiter_quant_op
|
||||
)
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
@@ -725,7 +734,11 @@ class RocmAiterSiluMulFp8GroupQuantFusionPass(VllmFusionPatternMatcherPass):
|
||||
def __init__(self, config: VllmConfig) -> None:
|
||||
super().__init__(config, "rocm_aiter_silu_mul_fp8_group_quant_fusion_pass")
|
||||
|
||||
self.register(AiterSiluMulFp8GroupQuantPattern())
|
||||
match_aiter_quant_op = not rocm_aiter_ops.is_rdna_aiter_enabled()
|
||||
|
||||
self.register(
|
||||
AiterSiluMulFp8GroupQuantPattern(match_aiter_quant_op=match_aiter_quant_op)
|
||||
)
|
||||
|
||||
self.dump_patterns(config, self.pm_pass)
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ from .ir.clone_elimination import UnsafeCloneEliminationPass
|
||||
from .ir.lowering_pass import VllmIRLoweringPass
|
||||
from .vllm_inductor_pass import VllmInductorPass, VllmPatternMatcherPass
|
||||
|
||||
if rocm_aiter_ops.is_enabled():
|
||||
if rocm_aiter_ops.is_enabled() or rocm_aiter_ops.is_rdna_aiter_enabled():
|
||||
from .fusion.allreduce_rms_fusion import (
|
||||
RocmAiterAllReduceFusionPass,
|
||||
)
|
||||
@@ -166,7 +166,9 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
|
||||
if enable_transformers_norm_canonicalization:
|
||||
self.passes += [AddRMSNormFusionPass(config)]
|
||||
|
||||
if self.pass_config.fuse_act_padding and rocm_aiter_ops.is_enabled():
|
||||
if self.pass_config.fuse_act_padding and (
|
||||
rocm_aiter_ops.is_enabled() or rocm_aiter_ops.is_rdna_aiter_enabled()
|
||||
):
|
||||
# Run the more specific RMSNorm+router-pad fusion before
|
||||
# AR+RMS, since both consume fused_add_rms_norm.
|
||||
self.passes += [RocmAiterTritonAddRMSNormPadFusionPass(config)]
|
||||
@@ -183,7 +185,10 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
|
||||
self.passes += [RMSNormReshapeFusionPass(config)]
|
||||
|
||||
if self.pass_config.fuse_norm_quant:
|
||||
if rocm_aiter_ops.is_enabled():
|
||||
if (
|
||||
rocm_aiter_ops.is_enabled()
|
||||
or rocm_aiter_ops.is_rdna_aiter_enabled()
|
||||
):
|
||||
self.passes += [
|
||||
RocmAiterRMSNormQuantFusionPass(config),
|
||||
]
|
||||
@@ -191,7 +196,10 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
|
||||
|
||||
if self.pass_config.fuse_act_quant:
|
||||
self.passes += [ActivationQuantFusionPass(config)]
|
||||
if rocm_aiter_ops.is_enabled():
|
||||
if (
|
||||
rocm_aiter_ops.is_enabled()
|
||||
or rocm_aiter_ops.is_rdna_aiter_enabled()
|
||||
):
|
||||
self.passes += [RocmAiterSiluMulFp8GroupQuantFusionPass(config)]
|
||||
|
||||
if self.pass_config.fuse_qk_norm_rope_kvcache:
|
||||
|
||||
@@ -397,10 +397,14 @@ class AiterFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
|
||||
@classmethod
|
||||
def is_supported(cls, compute_capability=None):
|
||||
if (
|
||||
rocm_aiter_ops.is_linear_enabled()
|
||||
or rocm_aiter_ops.is_rdna_linear_enabled()
|
||||
):
|
||||
return True, None
|
||||
return (
|
||||
rocm_aiter_ops.is_linear_enabled(),
|
||||
"Only supported on ROCm platform \
|
||||
with aiter package installed.",
|
||||
False,
|
||||
"Only supported on ROCm platform with aiter package installed.",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@@ -416,6 +420,17 @@ class AiterFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
"Supports only dynamic per token group activation "
|
||||
"quantization with group_shape=(1,128).",
|
||||
)
|
||||
|
||||
# RDNA4 (gfx12) only has the aiter Triton blockscale backend, which
|
||||
# needs a per-(N,K) tune. Reject untuned shapes so the dispatcher falls
|
||||
# through to the generic backend.
|
||||
if rocm_aiter_ops.is_rdna_linear_enabled():
|
||||
n, k = config.weight_shape
|
||||
if not rocm_aiter_ops.is_triton_gemm_w8a8_tuned(n, k):
|
||||
return (
|
||||
False,
|
||||
f"(N={n}, K={k}) is not in the aiter Triton blockscale tuned list.",
|
||||
)
|
||||
return True, None
|
||||
|
||||
def apply_block_scaled_mm(
|
||||
|
||||
@@ -108,10 +108,10 @@ class RowWiseTorchFP8ScaledMMLinearKernel(TorchFP8ScaledMMLinearKernel):
|
||||
if not current_platform.is_rocm():
|
||||
return False, "requires ROCm."
|
||||
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
from vllm.platforms.rocm import get_cdna_version, on_rdna4
|
||||
|
||||
if get_cdna_version() <= 2:
|
||||
return False, "requires CDNA3+"
|
||||
if get_cdna_version() <= 2 and not on_rdna4():
|
||||
return False, "requires CDNA3+ or RDNA4"
|
||||
|
||||
if compute_capability is not None and compute_capability < 94:
|
||||
return False, "requires compute capability 94 and above."
|
||||
|
||||
@@ -814,13 +814,13 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular):
|
||||
) -> bool:
|
||||
p = current_platform
|
||||
if p.is_rocm():
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
from vllm.platforms.rocm import get_cdna_version, on_rdna4
|
||||
|
||||
_rocm_support_fp8 = get_cdna_version() > 2
|
||||
is_rocm_fp8_supported = get_cdna_version() > 2 or on_rdna4()
|
||||
else:
|
||||
_rocm_support_fp8 = False
|
||||
is_rocm_fp8_supported = False
|
||||
|
||||
device_supports_fp8 = _rocm_support_fp8 or (
|
||||
device_supports_fp8 = is_rocm_fp8_supported or (
|
||||
p.is_cuda() and p.has_device_capability((8, 9))
|
||||
)
|
||||
|
||||
|
||||
@@ -379,8 +379,14 @@ def select_fp8_moe_backend(
|
||||
|
||||
# Handle explicit AITER FP8 configuration.
|
||||
if envs.is_set("VLLM_ROCM_USE_AITER") or envs.is_set("VLLM_ROCM_USE_AITER_MOE"):
|
||||
if not envs.VLLM_ROCM_USE_AITER or not envs.VLLM_ROCM_USE_AITER_MOE:
|
||||
AVAILABLE_BACKENDS.remove(Fp8MoeBackend.AITER)
|
||||
skip_aiter_moe = (
|
||||
not envs.VLLM_ROCM_USE_AITER
|
||||
or not envs.VLLM_ROCM_USE_AITER_MOE
|
||||
or rocm_aiter_ops.is_rdna_aiter_enabled()
|
||||
)
|
||||
if skip_aiter_moe:
|
||||
if Fp8MoeBackend.AITER in AVAILABLE_BACKENDS:
|
||||
AVAILABLE_BACKENDS.remove(Fp8MoeBackend.AITER)
|
||||
else:
|
||||
backend = Fp8MoeBackend.AITER
|
||||
return _return_or_raise(
|
||||
|
||||
@@ -298,7 +298,12 @@ def select_unquantized_moe_backend(
|
||||
|
||||
# Handle explicit AITER FP8 configuration.
|
||||
if envs.is_set("VLLM_ROCM_USE_AITER") or envs.is_set("VLLM_ROCM_USE_AITER_MOE"):
|
||||
if not envs.VLLM_ROCM_USE_AITER or not envs.VLLM_ROCM_USE_AITER_MOE:
|
||||
skip_aiter_moe = (
|
||||
not envs.VLLM_ROCM_USE_AITER
|
||||
or not envs.VLLM_ROCM_USE_AITER_MOE
|
||||
or rocm_aiter_ops.is_rdna_aiter_enabled()
|
||||
)
|
||||
if skip_aiter_moe:
|
||||
if UnquantizedMoeBackend.AITER in AVAILABLE_BACKENDS:
|
||||
AVAILABLE_BACKENDS.remove(UnquantizedMoeBackend.AITER)
|
||||
else:
|
||||
|
||||
@@ -69,7 +69,10 @@ from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
|
||||
# Availability is checked centrally via rocm_aiter_ops; the actual function
|
||||
# references are imported here so that they can be called without per-call
|
||||
# import overhead.
|
||||
GDN_AITER_TRITON_AVAILABLE = rocm_aiter_ops.are_gdn_triton_kernels_available()
|
||||
GDN_AITER_TRITON_AVAILABLE = (
|
||||
rocm_aiter_ops.are_gdn_triton_kernels_available()
|
||||
or rocm_aiter_ops.is_rdna_gdn_triton_kernels_available()
|
||||
)
|
||||
|
||||
if GDN_AITER_TRITON_AVAILABLE:
|
||||
from aiter.ops.triton.causal_conv1d_update_single_token import (
|
||||
|
||||
@@ -850,7 +850,7 @@ class SparseAttnIndexer(CustomOp):
|
||||
assert isinstance(q_quant, torch.Tensor), (
|
||||
"AMD sparse_attn_indexer expects a single FP8 q_quant tensor"
|
||||
)
|
||||
if rocm_aiter_ops.is_enabled():
|
||||
if rocm_aiter_ops.is_enabled() or rocm_aiter_ops.is_rdna_aiter_enabled():
|
||||
return torch.ops.vllm.rocm_aiter_sparse_attn_indexer(
|
||||
hidden_states,
|
||||
_encode_layer_name(self.k_cache.prefix),
|
||||
|
||||
@@ -218,6 +218,7 @@ _ON_GFX1250 = "gfx1250" in _GCN_ARCH
|
||||
_ON_CDNA = any(arch in _GCN_ARCH for arch in ["gfx9", "gfx1250"])
|
||||
# RDNA = gfx11/gfx12 minus the CDNA-classified gfx1250.
|
||||
_ON_RDNA = _ON_GFX1X and not _ON_CDNA
|
||||
_ON_RDNA4 = any(arch in _GCN_ARCH for arch in ["gfx1200", "gfx1201"])
|
||||
|
||||
|
||||
def _capability_from_gcn_arch(gcn_arch: str) -> tuple[int, int] | None:
|
||||
@@ -315,6 +316,10 @@ def on_gfx1250() -> bool:
|
||||
return _ON_GFX1250
|
||||
|
||||
|
||||
def on_rdna4() -> bool:
|
||||
return _ON_RDNA4
|
||||
|
||||
|
||||
def on_mi3xx() -> bool:
|
||||
return _ON_MI3XX
|
||||
|
||||
@@ -471,6 +476,8 @@ def _get_backend_priorities(
|
||||
backends.append(AttentionBackendEnum.ROCM_AITER_FA)
|
||||
if is_aiter_found_and_supported():
|
||||
backends.append(AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN)
|
||||
elif rocm_aiter_ops.is_rdna_aiter_enabled():
|
||||
backends.insert(0, AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN)
|
||||
backends.append(AttentionBackendEnum.TRITON_ATTN)
|
||||
backends.append(AttentionBackendEnum.TURBOQUANT)
|
||||
|
||||
@@ -918,7 +925,7 @@ class RocmPlatform(Platform):
|
||||
|
||||
@classmethod
|
||||
def supports_fp8(cls) -> bool:
|
||||
return on_cdna() or on_gfx12x()
|
||||
return on_cdna() or on_rdna4()
|
||||
|
||||
@classmethod
|
||||
def is_fp8_fnuz(cls) -> bool:
|
||||
@@ -1065,6 +1072,7 @@ class RocmPlatform(Platform):
|
||||
cc.cudagraph_mode != CUDAGraphMode.NONE
|
||||
and envs.VLLM_ROCM_USE_AITER
|
||||
and envs.VLLM_ROCM_USE_AITER_RMSNORM
|
||||
and not on_rdna4()
|
||||
):
|
||||
rms_norm = ["aiter"] + default
|
||||
else:
|
||||
|
||||
@@ -417,7 +417,7 @@ def rocm_fp8_paged_mqa_logits(
|
||||
batch_size, next_n = q_fp8.shape[:2]
|
||||
block_size = kv_cache_fp8.shape[1]
|
||||
|
||||
if rocm_aiter_ops.is_enabled():
|
||||
if rocm_aiter_ops.is_enabled() or rocm_aiter_ops.is_rdna_aiter_enabled():
|
||||
aiter_paged_mqa_logits_module = paged_mqa_logits_module()
|
||||
|
||||
if aiter_paged_mqa_logits_module is not None:
|
||||
@@ -582,7 +582,7 @@ def rocm_fp8_mqa_logits(
|
||||
)
|
||||
|
||||
aiter_mqa_logits_module = None
|
||||
if rocm_aiter_ops.is_enabled():
|
||||
if rocm_aiter_ops.is_enabled() or rocm_aiter_ops.is_rdna_aiter_enabled():
|
||||
aiter_mqa_logits_module = mqa_logits_module()
|
||||
|
||||
if aiter_mqa_logits_module is not None:
|
||||
|
||||
Reference in New Issue
Block a user