[Kernel] Add w8a8 CUTLASS kernels (#4749)

This commit is contained in:
Tyler Michael Smith
2024-05-16 18:32:50 -04:00
committed by GitHub
parent 8435b207af
commit 2060e93659
10 changed files with 1197 additions and 2 deletions
+17 -1
View File
@@ -1,4 +1,4 @@
from typing import Optional, Tuple
from typing import Optional, Tuple, Type
import torch
@@ -163,6 +163,22 @@ def gptq_marlin_24_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
size_k)
# cutlass
def cutlass_scaled_mm_dq(a: torch.Tensor, b: torch.Tensor,
a_scales: torch.Tensor, b_scales: torch.Tensor,
out_dtype: Type[torch.dtype]) -> torch.Tensor:
assert (b.shape[0] % 16 == 0 and b.shape[1] % 16 == 0)
assert (out_dtype is torch.bfloat16 or out_dtype is torch.float16)
m = a.shape[0]
n = b.shape[1]
out = torch.empty((m, n), dtype=out_dtype, device=a.device)
vllm_ops.cutlass_scaled_mm_dq(out, a, b, a_scales, b_scales)
return out
# aqlm
def aqlm_gemm(input: torch.Tensor, codes: torch.Tensor,
codebooks: torch.Tensor, scales: torch.Tensor,