[ROCm] Refactor tuned gemms (#55001)

Signed-off-by: Andy Friedrich <afriedri@amd.com>
Signed-off-by: afriedri <afriedri@amd.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
Co-authored-by: Shanshan Shen <467638484@qq.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
afriedri
2026-09-21 16:30:16 +00:00
committed by GitHub
co-authored by Douglas Lehr Shanshan Shen mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 8c0825090d
commit 874a6b5a06
2 changed files with 52 additions and 99 deletions
+3 -11
View File
@@ -50,16 +50,7 @@ SKINNY_GEMM_PASS_RATES = {
}
PRESHUFFLED_SHAPES = [
(64, 4096, 8192),
# aiter 0.1.20 Triton preshuffled fp4 GEMM OOBs (GPU memory access fault ->
# SIGABRT, crashing the process) for (M=32, N=8192, K=8192) on gfx950, despite
# advertising the (N, K) as tuned. A hard GPU fault can't be xfail'd, so skip it
# until the aiter kernel is fixed (ROCm/aiter#4867).
pytest.param(
(32, 8192, 8192),
marks=pytest.mark.skip(
reason="aiter 0.1.20 preshuffled fp4 GEMM OOB (M32,N8192,K8192, gfx950)"
),
),
(32, 8192, 16384),
]
@@ -430,7 +421,8 @@ def test_aiter_fp4_gemm_preshuffled_tuned_shapes(shape):
M, K, N = shape
assert M <= 64
assert rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(N, K)
# The predicate takes K as packed bytes, matching `weight.shape[1]`.
assert rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(N, K // 2)
A = torch.randn(M, K, dtype=torch.bfloat16)
B = torch.randn(N, K, dtype=torch.bfloat16)
+49 -88
View File
@@ -13,7 +13,6 @@ from torch._ops import OpOverload
import vllm.envs as envs
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.import_utils import PlaceholderModule
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
rocm_aiter_sparse_attn_indexer,
@@ -25,11 +24,6 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
try:
import pandas as pd
except ImportError:
pd = PlaceholderModule("pandas")
# fp8_dtype is not cached.
# on ROCm the fp8_dtype always calls is_fp8_fnuz
# which is a host op, so we cache it once here.
@@ -243,27 +237,42 @@ def is_aiter_found_and_supported_on_rdna4() -> bool:
return False
@functools.cache
def _load_gemm_tuned_configs(
q_dtype_w: torch.dtype, csv_path: str
) -> set[tuple[int, int, int]]:
try:
df = pd.read_csv(csv_path).drop_duplicates()
df = df[df["q_dtype_w"] == str(q_dtype_w)]
return set(zip(df["N"].astype(int), df["K"].astype(int), df["M"].astype(int)))
except Exception:
return set()
def _triton_gemm_config_is_tuned(config_name: str, N: int, K: int) -> bool:
from aiter.ops.triton.utils.gemm_config_utils import get_gemm_config
# any M shape under 1024 should be tuned
M_TUNE_PROBE = 128
return get_gemm_config(config_name, M_TUNE_PROBE, N, K)[1]
def _check_kernel_tuned(N: int, K: int, q_dtype_w: torch.dtype, csv_path: str) -> bool:
configs = _load_gemm_tuned_configs(q_dtype_w, csv_path)
def _ck_gemm_shape_is_tuned(
N: int, K: int, q_dtype_w: torch.dtype, csv_attr: str
) -> bool:
l_m = (
[1, 2, 4]
+ list(range(8, 513, 8))
+ [1024, 1536]
+ [2**i for i in range(11, 19)]
)
return any((N, K, M) in configs for M in l_m)
try:
from aiter.ops.gemm_op_a8w8 import (
AITER_CONFIGS,
get_GEMM_config_with_quant_type,
)
csv_path = getattr(AITER_CONFIGS, csv_attr)
return any(
get_GEMM_config_with_quant_type(M, N, K, q_dtype_w, csv_path) is not None
for M in l_m
)
except (AttributeError, ImportError, OSError):
logger.warning_once(
"Could not read aiter CK GEMM configs from AITER_CONFIGS.%s; "
"treating all shapes as untuned.",
csv_attr,
)
return False
def if_aiter_supported(func: Callable) -> Callable:
@@ -3458,89 +3467,41 @@ class rocm_aiter_ops:
)
@staticmethod
@functools.cache
def is_triton_gemm_w8a8_tuned(n: int, k: int) -> bool:
if not current_platform.is_rocm():
return False
from vllm.platforms.rocm import on_gfx950, on_rdna4
gfx950_tuned = {
(1024, 8192),
(2112, 7168),
(3072, 1536),
(32768, 8192),
(4096, 7168),
(4608, 7168),
(512, 7168),
(7168, 2048),
(7168, 256),
(8192, 1024),
(8192, 32768),
}
rdna4_tuned = gfx950_tuned | {
(2048, 2048),
(2624, 6144),
(3072, 6144),
(3584, 512),
(4096, 512),
(6144, 1536),
(6144, 2048),
(7168, 2304),
(7168, 16384),
(7168, 18432),
(8192, 8192),
(16384, 1536),
(24576, 1536),
(32768, 512),
(36864, 7168),
}
if on_rdna4():
return (n, k) in rdna4_tuned
if on_gfx950():
return (n, k) in gfx950_tuned
return False
try:
return _triton_gemm_config_is_tuned("GEMM-A8W8_BLOCKSCALE", n, k)
except (AssertionError, ImportError):
return False
@staticmethod
def is_triton_gemm_afp4wfp4_presh_ws_tuned(n: int, k: int) -> bool:
return (n, k) in [
(8192, 4096),
(1280, 8192),
(16384, 53248),
(106496, 16384),
(57344, 8192),
(8192, 2048),
(2560, 8192),
(10240, 8192),
(16384, 16384),
(8192, 28672),
(28672, 8192),
(18432, 16384),
(8192, 1024),
(7168, 8192),
(5120, 8192),
(8192, 8192),
(8192, 7168),
(14336, 8192),
(8192, 14336),
(8192, 3584),
]
@functools.cache
def is_triton_gemm_afp4wfp4_presh_ws_tuned(n: int, k_bytes: int) -> bool:
if not current_platform.is_rocm():
return False
# Input = weight.shape[1] (bytes); *4 for fp4 and aiter config 2*k
try:
return _triton_gemm_config_is_tuned(
"GEMM-AFP4WFP4_PRESHUFFLED", n, 4 * k_bytes
)
except (AssertionError, ImportError):
return False
@staticmethod
@functools.cache
def is_shuffled_per_token_w8a8_gemm_tuned(
N: int, K: int, q_dtype_w: torch.dtype
) -> bool:
import aiter.ops.gemm_op_a8w8 as aiter_gemm_a8w8_ops
csv_path = (
aiter_gemm_a8w8_ops.AITER_CONFIGS.AITER_CONFIG_GEMM_A8W8_BPRESHUFFLE_FILE
return _ck_gemm_shape_is_tuned(
N, K, q_dtype_w, "AITER_CONFIG_GEMM_A8W8_BPRESHUFFLE_FILE"
)
return _check_kernel_tuned(N, K, q_dtype_w, csv_path)
@staticmethod
@functools.cache
def is_per_token_w8a8_gemm_tuned(N: int, K: int, q_dtype_w: torch.dtype) -> bool:
import aiter.ops.gemm_op_a8w8 as aiter_gemm_a8w8_ops
csv_path = aiter_gemm_a8w8_ops.AITER_CONFIGS.AITER_CONFIG_GEMM_A8W8_FILE
return _check_kernel_tuned(N, K, q_dtype_w, csv_path)
return _ck_gemm_shape_is_tuned(N, K, q_dtype_w, "AITER_CONFIG_GEMM_A8W8_FILE")
@staticmethod
def shuffle_weight(