[Kernel] Add per-tensor and per-token AZP epilogues (#5941)
Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com>
This commit is contained in:
co-authored by
Tyler Michael Smith
parent
5c60c8c423
commit
8d59dbb000
+25
-1
@@ -241,6 +241,8 @@ def cutlass_scaled_mm(a: torch.Tensor,
|
||||
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
|
||||
assert (b.shape[0] % 16 == 0 and b.shape[1] % 16 == 0)
|
||||
assert (out_dtype is torch.bfloat16 or out_dtype is torch.float16)
|
||||
assert bias is None or bias.shape[0] == b.shape[
|
||||
1] and bias.dtype == out_dtype
|
||||
|
||||
m = a.shape[0]
|
||||
n = b.shape[1]
|
||||
@@ -251,6 +253,28 @@ def cutlass_scaled_mm(a: torch.Tensor,
|
||||
return out
|
||||
|
||||
|
||||
def cutlass_scaled_mm_azp(a: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
scale_a: torch.Tensor,
|
||||
scale_b: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
azp_adj: torch.Tensor,
|
||||
azp: Optional[torch.Tensor] = None,
|
||||
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
|
||||
assert (b.shape[0] % 16 == 0 and b.shape[1] % 16 == 0)
|
||||
assert (out_dtype is torch.bfloat16 or out_dtype is torch.float16)
|
||||
assert bias is None or bias.numel(
|
||||
) == b.shape[1] and bias.dtype == out_dtype
|
||||
|
||||
m = a.shape[0]
|
||||
n = b.shape[1]
|
||||
out = torch.empty((m, n), dtype=out_dtype, device=a.device)
|
||||
|
||||
torch.ops._C.cutlass_scaled_mm_azp(out, a, b, scale_a, scale_b, azp_adj,
|
||||
azp, bias)
|
||||
return out
|
||||
|
||||
|
||||
# aqlm
|
||||
def aqlm_gemm(input: torch.Tensor, codes: torch.Tensor,
|
||||
codebooks: torch.Tensor, scales: torch.Tensor,
|
||||
@@ -572,7 +596,7 @@ for k, v in names_and_values.items():
|
||||
if isinstance(v, fn_type) \
|
||||
and v.__code__.co_filename == __file__ \
|
||||
and any(arg is torch.Tensor or arg == "torch.Tensor"
|
||||
for arg in v.__annotations__.values()):
|
||||
for arg in v.__annotations__.values()):
|
||||
names_and_values_to_update[k] = hint_on_error(v)
|
||||
|
||||
names_and_values.update(names_and_values_to_update)
|
||||
|
||||
Reference in New Issue
Block a user