[ROCM] [FEAT] Integrate Aiter hipBLASLt GEMM online tuning (#40426)
Signed-off-by: hanlin12 <hanlin12@amd.com> Signed-off-by: Han Lin <hanlin12@amd.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
TJian
parent
c505cd93ef
commit
165b7864d0
@@ -128,6 +128,7 @@ from vllm.model_executor.kernels.linear.scaled_mm import (
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.aiter import (
|
||||
AiterFp8BlockScaledMMKernel,
|
||||
AiterHipbMMPerTokenFp8ScaledMMLinearKernel,
|
||||
AiterInt8ScaledMMLinearKernel,
|
||||
AiterPerTokenFp8ScaledMMLinearKernel,
|
||||
AiterPreshuffledPerTokenFp8ScaledMMLinearKernel,
|
||||
@@ -285,6 +286,7 @@ _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] =
|
||||
ChannelWiseTorchFP8ScaledMMLinearKernel,
|
||||
],
|
||||
PlatformEnum.ROCM: [
|
||||
AiterHipbMMPerTokenFp8ScaledMMLinearKernel,
|
||||
AiterPreshuffledPerTokenFp8ScaledMMLinearKernel,
|
||||
AiterPerTokenFp8ScaledMMLinearKernel,
|
||||
ROCmFP8ScaledMMLinearKernel,
|
||||
@@ -1024,6 +1026,7 @@ __all__ = [
|
||||
"FP8ScaledMMLinearLayerConfig",
|
||||
"Int8ScaledMMLinearLayerConfig",
|
||||
"ScaledMMLinearLayerConfig",
|
||||
"AiterHipbMMPerTokenFp8ScaledMMLinearKernel",
|
||||
"AiterPreshuffledPerTokenFp8ScaledMMLinearKernel",
|
||||
"AiterPerTokenFp8ScaledMMLinearKernel",
|
||||
"NvFp4LinearKernel",
|
||||
|
||||
@@ -212,6 +212,99 @@ class AiterPreshuffledPerTokenFp8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
)
|
||||
|
||||
|
||||
class AiterHipbMMPerTokenFp8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_rocm():
|
||||
return False, "requires ROCm."
|
||||
|
||||
if not rocm_aiter_ops.is_linear_hipbmm_enabled():
|
||||
return (
|
||||
False,
|
||||
"requires setting `VLLM_ROCM_USE_AITER=1`, "
|
||||
"`VLLM_ROCM_USE_AITER_LINEAR=1`, "
|
||||
"and `VLLM_ROCM_USE_AITER_LINEAR_HIPBMM=1`.",
|
||||
)
|
||||
try:
|
||||
import aiter # noqa: F401
|
||||
except Exception:
|
||||
return False, "requires aiter library to be installed."
|
||||
|
||||
if not hasattr(aiter, "hipb_mm"):
|
||||
return False, "requires aiter hipb_mm support."
|
||||
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: FP8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]:
|
||||
is_ptpc = (
|
||||
c.activation_quant_key.scale.group_shape.is_per_token()
|
||||
and c.weight_quant_key.scale.group_shape.is_per_channel()
|
||||
)
|
||||
if c.weight_shape is None:
|
||||
return False, "weight_shape is required for Aiter kernels"
|
||||
N, K = c.weight_shape
|
||||
|
||||
if c.out_dtype is not torch.bfloat16:
|
||||
return False, "requires bfloat16 output dtype."
|
||||
|
||||
if not is_ptpc:
|
||||
return (
|
||||
False,
|
||||
"requires per token activation scales and per channel weight scales.",
|
||||
)
|
||||
|
||||
if not (N >= 16 and N % 16 == 0 and K % 16 == 0):
|
||||
return (
|
||||
False,
|
||||
"requires N >= 16 and both N and K divisible by 16, "
|
||||
f"received N={N} and K={K}.",
|
||||
)
|
||||
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
w_name, w_s_name, *_ = self.layer_param_names
|
||||
w, w_s, *_ = self._get_layer_params(layer)
|
||||
|
||||
# Pre-apply the transposes that used to live in
|
||||
# _rocm_aiter_hipb_mm_fp8_impl so the kernel can consume B/Bs directly.
|
||||
# The `.t()` on the shuffled weight is kept as a non-contiguous view —
|
||||
# materializing it with `.contiguous()` would re-arrange the bytes and
|
||||
# break the `bpreshuffle` layout.
|
||||
shuffled_w = rocm_aiter_ops.shuffle_weight(w.t().contiguous())
|
||||
replace_parameter(
|
||||
layer,
|
||||
w_name,
|
||||
torch.nn.Parameter(shuffled_w.t(), requires_grad=False),
|
||||
)
|
||||
|
||||
if w_s.ndim > 1:
|
||||
replace_parameter(
|
||||
layer,
|
||||
w_s_name,
|
||||
torch.nn.Parameter(w_s.t().contiguous(), requires_grad=False),
|
||||
)
|
||||
|
||||
def apply_scaled_mm(
|
||||
self,
|
||||
*,
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
As: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
output_shape: list,
|
||||
) -> torch.Tensor:
|
||||
output_shape[-1] = B.shape[1]
|
||||
return rocm_aiter_ops.hipb_mm_fp8(A, B, As, Bs, bias, out_dtype).view(
|
||||
*output_shape
|
||||
)
|
||||
|
||||
|
||||
class AiterPerTokenFp8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
@classmethod
|
||||
def is_supported(
|
||||
|
||||
Reference in New Issue
Block a user