[ROCm] Enable AITER and FP8 inference on GFX120x (#43615)
Signed-off-by: skysnow2001 <skysnow9285@gmail.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
co-authored by
Douglas Lehr
parent
4a3447d200
commit
f43e1d26e3
@@ -814,13 +814,13 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular):
|
||||
) -> bool:
|
||||
p = current_platform
|
||||
if p.is_rocm():
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
from vllm.platforms.rocm import get_cdna_version, on_rdna4
|
||||
|
||||
_rocm_support_fp8 = get_cdna_version() > 2
|
||||
is_rocm_fp8_supported = get_cdna_version() > 2 or on_rdna4()
|
||||
else:
|
||||
_rocm_support_fp8 = False
|
||||
is_rocm_fp8_supported = False
|
||||
|
||||
device_supports_fp8 = _rocm_support_fp8 or (
|
||||
device_supports_fp8 = is_rocm_fp8_supported or (
|
||||
p.is_cuda() and p.has_device_capability((8, 9))
|
||||
)
|
||||
|
||||
|
||||
@@ -379,8 +379,14 @@ def select_fp8_moe_backend(
|
||||
|
||||
# Handle explicit AITER FP8 configuration.
|
||||
if envs.is_set("VLLM_ROCM_USE_AITER") or envs.is_set("VLLM_ROCM_USE_AITER_MOE"):
|
||||
if not envs.VLLM_ROCM_USE_AITER or not envs.VLLM_ROCM_USE_AITER_MOE:
|
||||
AVAILABLE_BACKENDS.remove(Fp8MoeBackend.AITER)
|
||||
skip_aiter_moe = (
|
||||
not envs.VLLM_ROCM_USE_AITER
|
||||
or not envs.VLLM_ROCM_USE_AITER_MOE
|
||||
or rocm_aiter_ops.is_rdna_aiter_enabled()
|
||||
)
|
||||
if skip_aiter_moe:
|
||||
if Fp8MoeBackend.AITER in AVAILABLE_BACKENDS:
|
||||
AVAILABLE_BACKENDS.remove(Fp8MoeBackend.AITER)
|
||||
else:
|
||||
backend = Fp8MoeBackend.AITER
|
||||
return _return_or_raise(
|
||||
|
||||
@@ -298,7 +298,12 @@ def select_unquantized_moe_backend(
|
||||
|
||||
# Handle explicit AITER FP8 configuration.
|
||||
if envs.is_set("VLLM_ROCM_USE_AITER") or envs.is_set("VLLM_ROCM_USE_AITER_MOE"):
|
||||
if not envs.VLLM_ROCM_USE_AITER or not envs.VLLM_ROCM_USE_AITER_MOE:
|
||||
skip_aiter_moe = (
|
||||
not envs.VLLM_ROCM_USE_AITER
|
||||
or not envs.VLLM_ROCM_USE_AITER_MOE
|
||||
or rocm_aiter_ops.is_rdna_aiter_enabled()
|
||||
)
|
||||
if skip_aiter_moe:
|
||||
if UnquantizedMoeBackend.AITER in AVAILABLE_BACKENDS:
|
||||
AVAILABLE_BACKENDS.remove(UnquantizedMoeBackend.AITER)
|
||||
else:
|
||||
|
||||
@@ -69,7 +69,10 @@ from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
|
||||
# Availability is checked centrally via rocm_aiter_ops; the actual function
|
||||
# references are imported here so that they can be called without per-call
|
||||
# import overhead.
|
||||
GDN_AITER_TRITON_AVAILABLE = rocm_aiter_ops.are_gdn_triton_kernels_available()
|
||||
GDN_AITER_TRITON_AVAILABLE = (
|
||||
rocm_aiter_ops.are_gdn_triton_kernels_available()
|
||||
or rocm_aiter_ops.is_rdna_gdn_triton_kernels_available()
|
||||
)
|
||||
|
||||
if GDN_AITER_TRITON_AVAILABLE:
|
||||
from aiter.ops.triton.causal_conv1d_update_single_token import (
|
||||
|
||||
@@ -850,7 +850,7 @@ class SparseAttnIndexer(CustomOp):
|
||||
assert isinstance(q_quant, torch.Tensor), (
|
||||
"AMD sparse_attn_indexer expects a single FP8 q_quant tensor"
|
||||
)
|
||||
if rocm_aiter_ops.is_enabled():
|
||||
if rocm_aiter_ops.is_enabled() or rocm_aiter_ops.is_rdna_aiter_enabled():
|
||||
return torch.ops.vllm.rocm_aiter_sparse_attn_indexer(
|
||||
hidden_states,
|
||||
_encode_layer_name(self.k_cache.prefix),
|
||||
|
||||
Reference in New Issue
Block a user