Improvements to wvSplitKrc skinny GEMM solution (#34304)
Signed-off-by: Hashem Hashemi <hashem.hashemi@amd.com>
This commit is contained in:
@@ -129,10 +129,6 @@ def rocm_unquantized_gemm_impl(
|
||||
k = weight.shape[1]
|
||||
|
||||
cu_count = num_compute_units()
|
||||
if use_aiter_triton_gemm(n, m, k, x.dtype):
|
||||
from aiter.ops.triton.gemm_a16w16 import gemm_a16w16
|
||||
|
||||
return gemm_a16w16(x, weight, bias)
|
||||
|
||||
# Next ^2 of n
|
||||
N_p2 = 1 << (n - 1).bit_length()
|
||||
@@ -145,7 +141,10 @@ def rocm_unquantized_gemm_impl(
|
||||
# Given the above, how many CUs would we need?
|
||||
CuNeeded = rndup_cus * GrpsShrB
|
||||
# candidate for atomic reduce count splitk?
|
||||
fits_wvsplitkrc = CuNeeded <= cu_count
|
||||
fits_wvsplitkrc = (
|
||||
N_p2 * m * ((k + 512 - 1) // 512)
|
||||
) <= 128 * 1024 * 12 # deterministic
|
||||
fits_wvsplitkrc &= CuNeeded <= cu_count
|
||||
|
||||
use_skinny_reduce_counting = (
|
||||
envs.VLLM_ROCM_USE_SKINNY_GEMM
|
||||
@@ -157,13 +156,16 @@ def rocm_unquantized_gemm_impl(
|
||||
and k > 512
|
||||
and m % 16 == 0
|
||||
and fits_wvsplitkrc
|
||||
and x.is_contiguous()
|
||||
and weight.is_contiguous()
|
||||
)
|
||||
)
|
||||
if use_skinny_reduce_counting:
|
||||
x_view = x.reshape(-1, x.size(-1))
|
||||
out = ops.wvSplitKrc(weight, x_view, cu_count, bias)
|
||||
return out.reshape(*x.shape[:-1], weight.shape[0])
|
||||
return ops.wvSplitKrc(x, weight, cu_count, bias)
|
||||
|
||||
if use_aiter_triton_gemm(n, m, k, x.dtype):
|
||||
from aiter.ops.triton.gemm_a16w16 import gemm_a16w16
|
||||
|
||||
return gemm_a16w16(x, weight, bias)
|
||||
|
||||
use_skinny = (
|
||||
envs.VLLM_ROCM_USE_SKINNY_GEMM
|
||||
|
||||
Reference in New Issue
Block a user