[ROCm][FEAT] Integrate aiter gemm w8a8 ptpc (#33773)

Signed-off-by: vllmellm <vllm.ellm@embeddedllm.com>
This commit is contained in:
vllmellm
2026-04-16 09:55:29 +08:00
committed by GitHub
parent 343f65234b
commit 5f7fab881a
9 changed files with 279 additions and 23 deletions
+97 -9
View File
@@ -3,6 +3,7 @@
import functools
from collections.abc import Callable
import pandas as pd
import torch
from torch._ops import OpOverload
@@ -56,6 +57,29 @@ def is_aiter_found_and_supported() -> bool:
return False
@functools.cache
def _load_gemm_tuned_configs(
q_dtype_w: torch.dtype, csv_path: str
) -> set[tuple[int, int, int]]:
try:
df = pd.read_csv(csv_path).drop_duplicates()
df = df[df["q_dtype_w"] == str(q_dtype_w)]
return set(zip(df["N"].astype(int), df["K"].astype(int), df["M"].astype(int)))
except Exception:
return set()
def _check_kernel_tuned(N: int, K: int, q_dtype_w: torch.dtype, csv_path: str) -> bool:
configs = _load_gemm_tuned_configs(q_dtype_w, csv_path)
l_m = (
[1, 2, 4]
+ list(range(8, 513, 8))
+ [1024, 1536]
+ [2**i for i in range(11, 19)]
)
return any((N, K, M) in configs for M in l_m)
def if_aiter_supported(func: Callable) -> Callable:
"""Decorator that only executes the function if
ROCm AITER package is supported and enabled on gfx9 archs.
@@ -468,7 +492,7 @@ def _rocm_aiter_mla_decode_fwd_fake(
pass
def _rocm_aiter_gemm_a8w8_impl(
def _rocm_aiter_w8a8_gemm_impl(
A: torch.Tensor,
B: torch.Tensor,
As: torch.Tensor,
@@ -485,7 +509,7 @@ def _rocm_aiter_gemm_a8w8_impl(
return gemm_a8w8_CK(A, B, As, Bs, bias, output_dtype)
def _rocm_aiter_gemm_a8w8_fake(
def _rocm_aiter_w8a8_gemm_fake(
A: torch.Tensor,
B: torch.Tensor,
As: torch.Tensor,
@@ -499,6 +523,35 @@ def _rocm_aiter_gemm_a8w8_fake(
return Y
def _rocm_aiter_preshuffled_per_token_w8a8_gemm_impl(
A: torch.Tensor,
B: torch.Tensor,
As: torch.Tensor,
Bs: torch.Tensor,
bias: torch.Tensor | None = None,
output_dtype: torch.dtype = torch.float16,
) -> torch.Tensor:
from aiter import gemm_a8w8_bpreshuffle
output = gemm_a8w8_bpreshuffle(A, B, As, Bs, None, output_dtype)
if bias is not None:
output.add_(bias)
return output
def _rocm_aiter_preshuffled_per_token_w8a8_gemm_fake(
A: torch.Tensor,
B: torch.Tensor,
As: torch.Tensor,
Bs: torch.Tensor,
bias: torch.Tensor | None = None,
output_dtype: torch.dtype = torch.float16,
) -> torch.Tensor:
m = A.shape[0]
n = B.shape[0]
return torch.empty(m, n, dtype=output_dtype, device=A.device)
def _rocm_aiter_triton_gemm_a8w8_blockscale_impl(
A: torch.Tensor,
B: torch.Tensor,
@@ -1313,11 +1366,15 @@ class rocm_aiter_ops:
)
direct_register_custom_op(
op_name="rocm_aiter_gemm_a8w8",
op_func=_rocm_aiter_gemm_a8w8_impl,
mutates_args=[],
fake_impl=_rocm_aiter_gemm_a8w8_fake,
dispatch_key=current_platform.dispatch_key,
op_name="rocm_aiter_w8a8_gemm",
op_func=_rocm_aiter_w8a8_gemm_impl,
fake_impl=_rocm_aiter_w8a8_gemm_fake,
)
direct_register_custom_op(
op_name="_rocm_aiter_preshuffled_per_token_w8a8_gemm",
op_func=_rocm_aiter_preshuffled_per_token_w8a8_gemm_impl,
fake_impl=_rocm_aiter_preshuffled_per_token_w8a8_gemm_fake,
)
direct_register_custom_op(
@@ -1493,7 +1550,7 @@ class rocm_aiter_ops:
)
@staticmethod
def gemm_a8w8(
def w8a8_gemm(
A: torch.Tensor,
B: torch.Tensor,
As: torch.Tensor,
@@ -1501,7 +1558,20 @@ class rocm_aiter_ops:
bias: torch.Tensor | None = None,
output_dtype: torch.dtype = torch.float16,
) -> torch.Tensor:
return torch.ops.vllm.rocm_aiter_gemm_a8w8(A, B, As, Bs, bias, output_dtype)
return torch.ops.vllm.rocm_aiter_w8a8_gemm(A, B, As, Bs, bias, output_dtype)
@staticmethod
def preshuffled_per_token_w8a8_gemm(
A: torch.Tensor,
B: torch.Tensor,
As: torch.Tensor,
Bs: torch.Tensor,
bias: torch.Tensor | None = None,
output_dtype: torch.dtype = torch.float16,
) -> torch.Tensor:
return torch.ops.vllm._rocm_aiter_preshuffled_per_token_w8a8_gemm(
A, B, As, Bs, bias, output_dtype
)
@staticmethod
def triton_gemm_a8w8_blockscale(
@@ -1920,6 +1990,24 @@ class rocm_aiter_ops:
(8192, 3584),
]
@staticmethod
def is_shuffled_per_token_w8a8_gemm_tuned(
N: int, K: int, q_dtype_w: torch.dtype
) -> bool:
import aiter.ops.gemm_op_a8w8 as aiter_gemm_a8w8_ops
csv_path = (
aiter_gemm_a8w8_ops.AITER_CONFIGS.AITER_CONFIG_GEMM_A8W8_BPRESHUFFLE_FILE
)
return _check_kernel_tuned(N, K, q_dtype_w, csv_path)
@staticmethod
def is_per_token_w8a8_gemm_tuned(N: int, K: int, q_dtype_w: torch.dtype) -> bool:
import aiter.ops.gemm_op_a8w8 as aiter_gemm_a8w8_ops
csv_path = aiter_gemm_a8w8_ops.AITER_CONFIGS.AITER_CONFIG_GEMM_A8W8_FILE
return _check_kernel_tuned(N, K, q_dtype_w, csv_path)
@staticmethod
def shuffle_weight(
tensor: torch.Tensor, layout: tuple[int, int] = (16, 16)