[ROCm][Perf] Pad fp16/bf16 GEMM weight row stride for L2 cache (#55090)
Signed-off-by: Matthias Gehre <matthias.gehre@amd.com>
This commit is contained in:
@@ -285,6 +285,44 @@ def wvsplitkrc_dispatch(n: int, k: int, m: int, cu_count: int) -> tuple[int, boo
|
||||
return chunkk, fits
|
||||
|
||||
|
||||
# On gfx11 a weight row stride that is a multiple of 2048 B makes the multi-row global
|
||||
# loads of a GEMM collide on the L2/MALL channel hash. Offsetting the stride by one
|
||||
# 128 B cache line dodges the collision and keeps each row cache-line aligned.
|
||||
def cache_aliasing_pad_elems(k: int, element_size: int) -> int:
|
||||
"""Row-stride pad, in elements, that stops an ``[n, k]`` weight aliasing on gfx11,
|
||||
or 0 when the stride does not alias.
|
||||
|
||||
Args:
|
||||
k: Number of elements per weight row.
|
||||
element_size: Size of one element in bytes.
|
||||
|
||||
Returns:
|
||||
The number of elements to add to the row stride.
|
||||
|
||||
"""
|
||||
if (k * element_size) % 2048 != 0:
|
||||
return 0
|
||||
return 128 // element_size
|
||||
|
||||
|
||||
def maybe_pad_weight_avoid_cache_aliasing(weight: torch.Tensor) -> torch.Tensor:
|
||||
"""Offset a dense weight's row stride so it stops aliasing.
|
||||
|
||||
Returns a view with the original shape whose row stride is one cache line
|
||||
wider, or ``weight`` unchanged when the pad does not apply.
|
||||
"""
|
||||
from vllm.platforms.rocm import on_gfx11
|
||||
|
||||
if not (current_platform.is_rocm() and on_gfx11()):
|
||||
return weight
|
||||
if weight.dim() != 2 or not weight.is_contiguous():
|
||||
return weight
|
||||
pad = cache_aliasing_pad_elems(weight.shape[1], weight.element_size())
|
||||
if pad == 0:
|
||||
return weight
|
||||
return torch.nn.functional.pad(weight, (0, pad))[..., :-pad]
|
||||
|
||||
|
||||
def rocm_unquantized_gemm_impl(
|
||||
x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None = None
|
||||
) -> torch.Tensor:
|
||||
@@ -295,7 +333,7 @@ def rocm_unquantized_gemm_impl(
|
||||
|
||||
return linear_batch_invariant(x, weight, bias)
|
||||
|
||||
from vllm.platforms.rocm import on_gfx1x, on_gfx9, on_gfx950, on_gfx1250
|
||||
from vllm.platforms.rocm import on_gfx1x, on_gfx9, on_gfx11, on_gfx950, on_gfx1250
|
||||
|
||||
n = x.numel() // x.size(-1)
|
||||
m = weight.shape[0]
|
||||
@@ -337,29 +375,49 @@ def rocm_unquantized_gemm_impl(
|
||||
|
||||
return gemm_a16w16(x, weight, bias)
|
||||
|
||||
use_skinny = (
|
||||
skinny_gemm_enabled = (
|
||||
envs.VLLM_ROCM_USE_SKINNY_GEMM
|
||||
and (on_gfx9() or on_gfx1x())
|
||||
# build (gfx9/gfx11 ISA); fall back to torch GEMM there.
|
||||
# TODO GFX1250: Include once skinny GEMM is supported on gfx1250
|
||||
and x.dtype in [torch.float16, torch.bfloat16]
|
||||
and k % 8 == 0
|
||||
and skinny_operands_compatible
|
||||
)
|
||||
|
||||
if use_skinny:
|
||||
# wvSplitK allows row strides. Perf impact is only validated on gfx11.
|
||||
weight_ok_for_wvsplitk = weight.is_contiguous() or (
|
||||
on_gfx11() and weight.stride(1) == 1
|
||||
)
|
||||
|
||||
use_wvsplitk = (
|
||||
skinny_gemm_enabled
|
||||
and weight_ok_for_wvsplitk
|
||||
and (bias is None or bias.is_contiguous())
|
||||
and (m == 1 or m > 8)
|
||||
and 0 < n <= 5
|
||||
)
|
||||
|
||||
use_llmm1 = (
|
||||
skinny_gemm_enabled
|
||||
and weight.is_contiguous()
|
||||
and bias is None
|
||||
and m % 4 == 0
|
||||
and n == 1
|
||||
and k <= 8192
|
||||
)
|
||||
|
||||
if use_wvsplitk:
|
||||
# The skinny kernels assume contiguous K elements. A shape-preserving
|
||||
# reshape can retain a transposed activation's non-contiguous strides.
|
||||
# Note: Only build that view inside the branches that consume it.
|
||||
if (m == 1 or m > 8) and 0 < n <= 5:
|
||||
x_view = x.reshape(-1, x.size(-1)).contiguous()
|
||||
cu_count = num_compute_units()
|
||||
out = ops.wvSplitK(weight, x_view, cu_count, bias)
|
||||
return out.reshape(*x.shape[:-1], weight.shape[0])
|
||||
elif m % 4 == 0 and n == 1 and k <= 8192 and bias is None:
|
||||
x_view = x.reshape(-1, x.size(-1)).contiguous()
|
||||
out = ops.LLMM1(weight, x_view, 4)
|
||||
return out.reshape(*x.shape[:-1], weight.shape[0])
|
||||
x_view = x.reshape(-1, x.size(-1)).contiguous()
|
||||
out = ops.wvSplitK(weight, x_view, cu_count, bias)
|
||||
return out.reshape(*x.shape[:-1], weight.shape[0])
|
||||
|
||||
if use_llmm1:
|
||||
x_view = x.reshape(-1, x.size(-1)).contiguous()
|
||||
out = ops.LLMM1(weight, x_view, 4)
|
||||
return out.reshape(*x.shape[:-1], weight.shape[0])
|
||||
|
||||
if rocm_aiter_ops.is_tgemm_enabled():
|
||||
from aiter.tuned_gemm import tgemm
|
||||
|
||||
Reference in New Issue
Block a user