Signed-off-by: Gregory Shtrasberg <Gregory.Shtrasberg@amd.com>
This commit is contained in:
@@ -2027,35 +2027,20 @@ def selective_scan_fwd(
|
||||
)
|
||||
|
||||
|
||||
# NOTE: The wvSplitK kernel (and all of the kernels in skinny_gemms.cu)
|
||||
# are unable to properly handle non-contiguous
|
||||
# tensors. It might be a good TODO(rasmith) to augment these kernels
|
||||
# to be able to handle non-contiguous kernels for better performance.
|
||||
def rocm_enforce_contiguous_skinny_gemm_inputs(
|
||||
a: torch.Tensor, b: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
a = a.contiguous() # no-op if already contiguous, else clone
|
||||
b = b.contiguous() # no-op if already contiguous, else clone
|
||||
return a, b
|
||||
|
||||
|
||||
# ROCm skinny gemms
|
||||
def LLMM1(a: torch.Tensor, b: torch.Tensor, rows_per_block: int) -> torch.Tensor:
|
||||
a, b = rocm_enforce_contiguous_skinny_gemm_inputs(a, b)
|
||||
return torch.ops._rocm_C.LLMM1(a, b, rows_per_block)
|
||||
|
||||
|
||||
def wvSplitK(
|
||||
a: torch.Tensor, b: torch.Tensor, cu_count: int, bias: torch.Tensor = None
|
||||
) -> torch.Tensor:
|
||||
a, b = rocm_enforce_contiguous_skinny_gemm_inputs(a, b)
|
||||
return torch.ops._rocm_C.wvSplitK(a, b, bias, cu_count)
|
||||
|
||||
|
||||
def wvSplitKrc(
|
||||
a: torch.Tensor, b: torch.Tensor, cu_count: int, bias: torch.Tensor = None
|
||||
) -> torch.Tensor:
|
||||
a, b = rocm_enforce_contiguous_skinny_gemm_inputs(a, b)
|
||||
return torch.ops._rocm_C.wvSplitKrc(a, b, bias, cu_count)
|
||||
|
||||
|
||||
@@ -2068,7 +2053,6 @@ def wvSplitKQ(
|
||||
cu_count: int,
|
||||
bias: torch.Tensor = None,
|
||||
) -> torch.Tensor:
|
||||
a, b = rocm_enforce_contiguous_skinny_gemm_inputs(a, b)
|
||||
out = torch.empty((b.shape[0], a.shape[0]), dtype=out_dtype, device=b.device)
|
||||
torch.ops._rocm_C.wvSplitKQ(a, b, bias, out, scale_a, scale_b, cu_count)
|
||||
return out
|
||||
|
||||
@@ -28,6 +28,7 @@ def rocm_per_tensor_float_w8a8_scaled_mm_impl(
|
||||
A.shape[0] == 1
|
||||
and B.shape[1] % 16 == 0
|
||||
and ((bias is None) or (bias.dtype == out_dtype))
|
||||
and A.is_contiguous()
|
||||
):
|
||||
output = ops.wvSplitKQ(
|
||||
B.t(),
|
||||
|
||||
@@ -165,6 +165,7 @@ def rocm_unquantized_gemm_impl(
|
||||
and n <= 128
|
||||
and k > 512
|
||||
and math.ceil(k / 512) * math.ceil(m / 16) < get_cu_count()
|
||||
and x.is_contiguous()
|
||||
)
|
||||
# k == 2880 and (m == 640 or m == 128))
|
||||
)
|
||||
@@ -179,6 +180,7 @@ def rocm_unquantized_gemm_impl(
|
||||
and on_gfx9()
|
||||
and x.dtype in [torch.float16, torch.bfloat16]
|
||||
and k % 8 == 0
|
||||
and x.is_contiguous()
|
||||
)
|
||||
|
||||
if use_skinny is not True:
|
||||
|
||||
Reference in New Issue
Block a user