[ROCM] [FEAT] Integrate Aiter hipBLASLt GEMM online tuning (#40426)
Signed-off-by: hanlin12 <hanlin12@amd.com> Signed-off-by: Han Lin <hanlin12@amd.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
TJian
parent
c505cd93ef
commit
165b7864d0
@@ -27,6 +27,16 @@ except ImportError:
|
||||
# on ROCm the fp8_dtype always calls is_fp8_fnuz
|
||||
# which is a host op, so we cache it once here.
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
_HIPB_MM_INITIALIZED_DEVICES: set[int] = set()
|
||||
|
||||
|
||||
def _ensure_hipb_mm_extension_initialized() -> None:
|
||||
import aiter
|
||||
|
||||
device = torch.accelerator.current_device_index()
|
||||
if device not in _HIPB_MM_INITIALIZED_DEVICES:
|
||||
aiter.hipb_create_extension()
|
||||
_HIPB_MM_INITIALIZED_DEVICES.add(device)
|
||||
|
||||
|
||||
def is_aiter_found() -> bool:
|
||||
@@ -625,6 +635,43 @@ def _rocm_aiter_preshuffled_per_token_w8a8_gemm_fake(
|
||||
return torch.empty(m, n, dtype=output_dtype, device=A.device)
|
||||
|
||||
|
||||
def _rocm_aiter_hipb_mm_fp8_impl(
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
As: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
output_dtype: torch.dtype = torch.bfloat16,
|
||||
) -> torch.Tensor:
|
||||
from aiter import hipb_mm
|
||||
|
||||
_ensure_hipb_mm_extension_initialized()
|
||||
return hipb_mm(
|
||||
A,
|
||||
B,
|
||||
solution_index=-1,
|
||||
bias=bias,
|
||||
out_dtype=output_dtype,
|
||||
scaleA=As,
|
||||
scaleB=Bs,
|
||||
scaleOut=None,
|
||||
bpreshuffle=True,
|
||||
)
|
||||
|
||||
|
||||
def _rocm_aiter_hipb_mm_fp8_fake(
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
As: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
output_dtype: torch.dtype = torch.bfloat16,
|
||||
) -> torch.Tensor:
|
||||
m = A.shape[0]
|
||||
n = B.shape[1]
|
||||
return torch.empty(m, n, dtype=output_dtype, device=A.device)
|
||||
|
||||
|
||||
def _rocm_aiter_triton_gemm_a8w8_blockscale_impl(
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
@@ -1308,6 +1355,7 @@ class rocm_aiter_ops:
|
||||
# TODO: Consolidate under _LINEAR_ENABLED
|
||||
_FP8BMM_ENABLED = envs.VLLM_ROCM_USE_AITER_FP8BMM
|
||||
_FP4BMM_ENABLED = envs.VLLM_ROCM_USE_AITER_FP4BMM
|
||||
_LINEAR_HIPBMM_ENABLED = envs.VLLM_ROCM_USE_AITER_LINEAR_HIPBMM
|
||||
# TODO: Consolidate under _LINEAR_ENABLED
|
||||
_FP4_GEMM_DYNAMIC_QUANT_ASM = envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM
|
||||
# TODO: Consolidate under VLLM_ROCM_USE_AITER_ROPE
|
||||
@@ -1340,6 +1388,7 @@ class rocm_aiter_ops:
|
||||
cls._TRITON_UNIFIED_ATTN_ENABLED = envs.VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION
|
||||
cls._FP8BMM_ENABLED = envs.VLLM_ROCM_USE_AITER_FP8BMM
|
||||
cls._FP4BMM_ENABLED = envs.VLLM_ROCM_USE_AITER_FP4BMM
|
||||
cls._LINEAR_HIPBMM_ENABLED = envs.VLLM_ROCM_USE_AITER_LINEAR_HIPBMM
|
||||
cls._FP4_GEMM_DYNAMIC_QUANT_ASM = envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM
|
||||
cls._TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE
|
||||
cls._MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
|
||||
@@ -1512,6 +1561,13 @@ class rocm_aiter_ops:
|
||||
|
||||
return cls._AITER_ENABLED and cls._FP4BMM_ENABLED and on_gfx950()
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def is_linear_hipbmm_enabled(cls) -> bool:
|
||||
from vllm.platforms.rocm import on_mi3xx
|
||||
|
||||
return cls.is_linear_enabled() and on_mi3xx() and cls._LINEAR_HIPBMM_ENABLED
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def is_asm_fp4_gemm_dynamic_quant_enabled(cls) -> bool:
|
||||
@@ -1668,6 +1724,12 @@ class rocm_aiter_ops:
|
||||
fake_impl=_rocm_aiter_preshuffled_per_token_w8a8_gemm_fake,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="rocm_aiter_hipb_mm_fp8",
|
||||
op_func=_rocm_aiter_hipb_mm_fp8_impl,
|
||||
fake_impl=_rocm_aiter_hipb_mm_fp8_fake,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="rocm_aiter_triton_gemm_a8w8_blockscale",
|
||||
op_func=_rocm_aiter_triton_gemm_a8w8_blockscale_impl,
|
||||
@@ -1858,6 +1920,17 @@ class rocm_aiter_ops:
|
||||
A, B, As, Bs, bias, output_dtype
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def hipb_mm_fp8(
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
As: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
output_dtype: torch.dtype = torch.bfloat16,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops.vllm.rocm_aiter_hipb_mm_fp8(A, B, As, Bs, bias, output_dtype)
|
||||
|
||||
@staticmethod
|
||||
def triton_gemm_a8w8_blockscale(
|
||||
A: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user