[ROCM] [FEAT] Integrate Aiter hipBLASLt GEMM online tuning (#40426)

Signed-off-by: hanlin12 <hanlin12@amd.com>
Signed-off-by: Han Lin <hanlin12@amd.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
This commit is contained in:
Han Lin
2026-06-04 23:45:36 -07:00
committed by GitHub
co-authored by gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> TJian
parent c505cd93ef
commit 165b7864d0
6 changed files with 591 additions and 0 deletions
@@ -128,6 +128,7 @@ from vllm.model_executor.kernels.linear.scaled_mm import (
)
from vllm.model_executor.kernels.linear.scaled_mm.aiter import (
AiterFp8BlockScaledMMKernel,
AiterHipbMMPerTokenFp8ScaledMMLinearKernel,
AiterInt8ScaledMMLinearKernel,
AiterPerTokenFp8ScaledMMLinearKernel,
AiterPreshuffledPerTokenFp8ScaledMMLinearKernel,
@@ -285,6 +286,7 @@ _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] =
ChannelWiseTorchFP8ScaledMMLinearKernel,
],
PlatformEnum.ROCM: [
AiterHipbMMPerTokenFp8ScaledMMLinearKernel,
AiterPreshuffledPerTokenFp8ScaledMMLinearKernel,
AiterPerTokenFp8ScaledMMLinearKernel,
ROCmFP8ScaledMMLinearKernel,
@@ -1024,6 +1026,7 @@ __all__ = [
"FP8ScaledMMLinearLayerConfig",
"Int8ScaledMMLinearLayerConfig",
"ScaledMMLinearLayerConfig",
"AiterHipbMMPerTokenFp8ScaledMMLinearKernel",
"AiterPreshuffledPerTokenFp8ScaledMMLinearKernel",
"AiterPerTokenFp8ScaledMMLinearKernel",
"NvFp4LinearKernel",
@@ -212,6 +212,99 @@ class AiterPreshuffledPerTokenFp8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
)
class AiterHipbMMPerTokenFp8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_rocm():
return False, "requires ROCm."
if not rocm_aiter_ops.is_linear_hipbmm_enabled():
return (
False,
"requires setting `VLLM_ROCM_USE_AITER=1`, "
"`VLLM_ROCM_USE_AITER_LINEAR=1`, "
"and `VLLM_ROCM_USE_AITER_LINEAR_HIPBMM=1`.",
)
try:
import aiter # noqa: F401
except Exception:
return False, "requires aiter library to be installed."
if not hasattr(aiter, "hipb_mm"):
return False, "requires aiter hipb_mm support."
return True, None
@classmethod
def can_implement(cls, c: FP8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]:
is_ptpc = (
c.activation_quant_key.scale.group_shape.is_per_token()
and c.weight_quant_key.scale.group_shape.is_per_channel()
)
if c.weight_shape is None:
return False, "weight_shape is required for Aiter kernels"
N, K = c.weight_shape
if c.out_dtype is not torch.bfloat16:
return False, "requires bfloat16 output dtype."
if not is_ptpc:
return (
False,
"requires per token activation scales and per channel weight scales.",
)
if not (N >= 16 and N % 16 == 0 and K % 16 == 0):
return (
False,
"requires N >= 16 and both N and K divisible by 16, "
f"received N={N} and K={K}.",
)
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
w_name, w_s_name, *_ = self.layer_param_names
w, w_s, *_ = self._get_layer_params(layer)
# Pre-apply the transposes that used to live in
# _rocm_aiter_hipb_mm_fp8_impl so the kernel can consume B/Bs directly.
# The `.t()` on the shuffled weight is kept as a non-contiguous view —
# materializing it with `.contiguous()` would re-arrange the bytes and
# break the `bpreshuffle` layout.
shuffled_w = rocm_aiter_ops.shuffle_weight(w.t().contiguous())
replace_parameter(
layer,
w_name,
torch.nn.Parameter(shuffled_w.t(), requires_grad=False),
)
if w_s.ndim > 1:
replace_parameter(
layer,
w_s_name,
torch.nn.Parameter(w_s.t().contiguous(), requires_grad=False),
)
def apply_scaled_mm(
self,
*,
A: torch.Tensor,
B: torch.Tensor,
out_dtype: torch.dtype,
As: torch.Tensor,
Bs: torch.Tensor,
bias: torch.Tensor | None,
output_shape: list,
) -> torch.Tensor:
output_shape[-1] = B.shape[1]
return rocm_aiter_ops.hipb_mm_fp8(A, B, As, Bs, bias, out_dtype).view(
*output_shape
)
class AiterPerTokenFp8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
@classmethod
def is_supported(