[ROCm] Enable AITER and FP8 inference on GFX120x (#43615)
Signed-off-by: skysnow2001 <skysnow9285@gmail.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
co-authored by
Douglas Lehr
parent
4a3447d200
commit
f43e1d26e3
@@ -397,10 +397,14 @@ class AiterFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
|
||||
@classmethod
|
||||
def is_supported(cls, compute_capability=None):
|
||||
if (
|
||||
rocm_aiter_ops.is_linear_enabled()
|
||||
or rocm_aiter_ops.is_rdna_linear_enabled()
|
||||
):
|
||||
return True, None
|
||||
return (
|
||||
rocm_aiter_ops.is_linear_enabled(),
|
||||
"Only supported on ROCm platform \
|
||||
with aiter package installed.",
|
||||
False,
|
||||
"Only supported on ROCm platform with aiter package installed.",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@@ -416,6 +420,17 @@ class AiterFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
"Supports only dynamic per token group activation "
|
||||
"quantization with group_shape=(1,128).",
|
||||
)
|
||||
|
||||
# RDNA4 (gfx12) only has the aiter Triton blockscale backend, which
|
||||
# needs a per-(N,K) tune. Reject untuned shapes so the dispatcher falls
|
||||
# through to the generic backend.
|
||||
if rocm_aiter_ops.is_rdna_linear_enabled():
|
||||
n, k = config.weight_shape
|
||||
if not rocm_aiter_ops.is_triton_gemm_w8a8_tuned(n, k):
|
||||
return (
|
||||
False,
|
||||
f"(N={n}, K={k}) is not in the aiter Triton blockscale tuned list.",
|
||||
)
|
||||
return True, None
|
||||
|
||||
def apply_block_scaled_mm(
|
||||
|
||||
@@ -108,10 +108,10 @@ class RowWiseTorchFP8ScaledMMLinearKernel(TorchFP8ScaledMMLinearKernel):
|
||||
if not current_platform.is_rocm():
|
||||
return False, "requires ROCm."
|
||||
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
from vllm.platforms.rocm import get_cdna_version, on_rdna4
|
||||
|
||||
if get_cdna_version() <= 2:
|
||||
return False, "requires CDNA3+"
|
||||
if get_cdna_version() <= 2 and not on_rdna4():
|
||||
return False, "requires CDNA3+ or RDNA4"
|
||||
|
||||
if compute_capability is not None and compute_capability < 94:
|
||||
return False, "requires compute capability 94 and above."
|
||||
|
||||
Reference in New Issue
Block a user