[ROCm][Perf] Pad fp16/bf16 GEMM weight row stride for L2 cache (#55090)

Signed-off-by: Matthias Gehre <matthias.gehre@amd.com>
This commit is contained in:
Matthias Gehre
2026-10-15 14:26:32 +00:00
committed by GitHub
parent d12d5460a5
commit 011fa68ca9
3 changed files with 198 additions and 13 deletions
+71 -13
View File
@@ -285,6 +285,44 @@ def wvsplitkrc_dispatch(n: int, k: int, m: int, cu_count: int) -> tuple[int, boo
return chunkk, fits
# On gfx11 a weight row stride that is a multiple of 2048 B makes the multi-row global
# loads of a GEMM collide on the L2/MALL channel hash. Offsetting the stride by one
# 128 B cache line dodges the collision and keeps each row cache-line aligned.
def cache_aliasing_pad_elems(k: int, element_size: int) -> int:
"""Row-stride pad, in elements, that stops an ``[n, k]`` weight aliasing on gfx11,
or 0 when the stride does not alias.
Args:
k: Number of elements per weight row.
element_size: Size of one element in bytes.
Returns:
The number of elements to add to the row stride.
"""
if (k * element_size) % 2048 != 0:
return 0
return 128 // element_size
def maybe_pad_weight_avoid_cache_aliasing(weight: torch.Tensor) -> torch.Tensor:
"""Offset a dense weight's row stride so it stops aliasing.
Returns a view with the original shape whose row stride is one cache line
wider, or ``weight`` unchanged when the pad does not apply.
"""
from vllm.platforms.rocm import on_gfx11
if not (current_platform.is_rocm() and on_gfx11()):
return weight
if weight.dim() != 2 or not weight.is_contiguous():
return weight
pad = cache_aliasing_pad_elems(weight.shape[1], weight.element_size())
if pad == 0:
return weight
return torch.nn.functional.pad(weight, (0, pad))[..., :-pad]
def rocm_unquantized_gemm_impl(
x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None = None
) -> torch.Tensor:
@@ -295,7 +333,7 @@ def rocm_unquantized_gemm_impl(
return linear_batch_invariant(x, weight, bias)
from vllm.platforms.rocm import on_gfx1x, on_gfx9, on_gfx950, on_gfx1250
from vllm.platforms.rocm import on_gfx1x, on_gfx9, on_gfx11, on_gfx950, on_gfx1250
n = x.numel() // x.size(-1)
m = weight.shape[0]
@@ -337,29 +375,49 @@ def rocm_unquantized_gemm_impl(
return gemm_a16w16(x, weight, bias)
use_skinny = (
skinny_gemm_enabled = (
envs.VLLM_ROCM_USE_SKINNY_GEMM
and (on_gfx9() or on_gfx1x())
# build (gfx9/gfx11 ISA); fall back to torch GEMM there.
# TODO GFX1250: Include once skinny GEMM is supported on gfx1250
and x.dtype in [torch.float16, torch.bfloat16]
and k % 8 == 0
and skinny_operands_compatible
)
if use_skinny:
# wvSplitK allows row strides. Perf impact is only validated on gfx11.
weight_ok_for_wvsplitk = weight.is_contiguous() or (
on_gfx11() and weight.stride(1) == 1
)
use_wvsplitk = (
skinny_gemm_enabled
and weight_ok_for_wvsplitk
and (bias is None or bias.is_contiguous())
and (m == 1 or m > 8)
and 0 < n <= 5
)
use_llmm1 = (
skinny_gemm_enabled
and weight.is_contiguous()
and bias is None
and m % 4 == 0
and n == 1
and k <= 8192
)
if use_wvsplitk:
# The skinny kernels assume contiguous K elements. A shape-preserving
# reshape can retain a transposed activation's non-contiguous strides.
# Note: Only build that view inside the branches that consume it.
if (m == 1 or m > 8) and 0 < n <= 5:
x_view = x.reshape(-1, x.size(-1)).contiguous()
cu_count = num_compute_units()
out = ops.wvSplitK(weight, x_view, cu_count, bias)
return out.reshape(*x.shape[:-1], weight.shape[0])
elif m % 4 == 0 and n == 1 and k <= 8192 and bias is None:
x_view = x.reshape(-1, x.size(-1)).contiguous()
out = ops.LLMM1(weight, x_view, 4)
return out.reshape(*x.shape[:-1], weight.shape[0])
x_view = x.reshape(-1, x.size(-1)).contiguous()
out = ops.wvSplitK(weight, x_view, cu_count, bias)
return out.reshape(*x.shape[:-1], weight.shape[0])
if use_llmm1:
x_view = x.reshape(-1, x.size(-1)).contiguous()
out = ops.LLMM1(weight, x_view, 4)
return out.reshape(*x.shape[:-1], weight.shape[0])
if rocm_aiter_ops.is_tgemm_enabled():
from aiter.tuned_gemm import tgemm