[Transform] [Quantization] Add QuTLASS support to vLLM (#24440)
Signed-off-by: LopezCastroRoberto <roberto.lopez.castro@udc.es> Signed-off-by: Roberto L. Castro <38211239+LopezCastroRoberto@users.noreply.github.com> Signed-off-by: Andrei Panferov <andrei@panferov.org> Co-authored-by: Andrei Panferov <andrei@panferov.org> Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Andrei Panferov
Michael Goin
parent
8d2b8c0ff2
commit
96ad65b7fe
+139
-1
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING, Optional, Union
|
||||
from typing import TYPE_CHECKING, Literal, Optional, Union
|
||||
|
||||
import torch
|
||||
|
||||
@@ -2483,6 +2483,144 @@ def onednn_scaled_mm(
|
||||
return output
|
||||
|
||||
|
||||
if hasattr(torch.ops._qutlass_C, "matmul_mxf4_bf16_tn"):
|
||||
|
||||
@register_fake("_qutlass_C::matmul_mxf4_bf16_tn")
|
||||
def _fake_matmul_mxf4_bf16_tn(
|
||||
a: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
a_sf: torch.Tensor,
|
||||
b_sf: torch.Tensor,
|
||||
alpha: torch.Tensor,
|
||||
):
|
||||
return a.new_empty(*a.shape[:-1], b.shape[0], dtype=torch.bfloat16)
|
||||
|
||||
|
||||
def matmul_mxf4_bf16_tn(
|
||||
a: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
a_sf: torch.Tensor,
|
||||
b_sf: torch.Tensor,
|
||||
alpha: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops._qutlass_C.matmul_mxf4_bf16_tn(a, b, a_sf, b_sf, alpha)
|
||||
|
||||
|
||||
if hasattr(torch.ops._qutlass_C, "matmul_ada_mxf4_bf16_tn"):
|
||||
|
||||
@register_fake("_qutlass_C::matmul_ada_mxf4_bf16_tn")
|
||||
def _fake_matmul_ada_mxf4_bf16_tn(
|
||||
a: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
a_sf: torch.Tensor,
|
||||
b_sf: torch.Tensor,
|
||||
alpha: torch.Tensor,
|
||||
):
|
||||
return a.new_empty(*a.shape[:-1], b.shape[0], dtype=torch.bfloat16)
|
||||
|
||||
|
||||
def matmul_ada_mxf4_bf16_tn(
|
||||
a: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
a_sf: torch.Tensor,
|
||||
b_sf: torch.Tensor,
|
||||
alpha: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops._qutlass_C.matmul_ada_mxf4_bf16_tn(a, b, a_sf, b_sf, alpha)
|
||||
|
||||
|
||||
def ceil_div(a, b):
|
||||
return (a + b - 1) // b
|
||||
|
||||
|
||||
if hasattr(torch.ops._qutlass_C, "fusedQuantizeMxQuest"):
|
||||
|
||||
@register_fake("_qutlass_C::fusedQuantizeMxQuest")
|
||||
def _fake_fused_quantize_mx_quest(
|
||||
a: torch.Tensor, b: torch.Tensor, xh_e2m1: torch.Tensor, xh_e8m0: torch.Tensor
|
||||
):
|
||||
return xh_e2m1, xh_e8m0
|
||||
|
||||
|
||||
if hasattr(torch.ops._qutlass_C, "fusedQuantizeMxAbsMax"):
|
||||
|
||||
@register_fake("_qutlass_C::fusedQuantizeMxAbsMax")
|
||||
def _fake_fused_quantize_mx_absmax(
|
||||
a: torch.Tensor, b: torch.Tensor, xh_e2m1: torch.Tensor, xh_e8m0: torch.Tensor
|
||||
):
|
||||
return xh_e2m1, xh_e8m0
|
||||
|
||||
|
||||
def fusedQuantizeMx(
|
||||
a: torch.Tensor, b: torch.Tensor, *, method: Literal["quest", "abs_max"] = "quest"
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if a.dim() == 0:
|
||||
raise ValueError("`a` must have at least 1 dimension.")
|
||||
if a.size(-1) % 32 != 0:
|
||||
raise ValueError(f"last dim of `a` must be divisible by 32, got {a.size(-1)}.")
|
||||
if b.device != a.device:
|
||||
raise ValueError("`a` and `b` must be on the same device.")
|
||||
|
||||
xh_e2m1 = torch.empty(
|
||||
*a.shape[:-1], a.size(-1) // 2, dtype=torch.uint8, device=a.device
|
||||
)
|
||||
|
||||
rows, cols = a.numel() // a.size(-1), a.size(-1) // 32
|
||||
n_row_blocks = ceil_div(rows, 128)
|
||||
n_col_blocks = ceil_div(cols, 4)
|
||||
padded_rows = n_row_blocks * 128
|
||||
padded_cols = n_col_blocks * 4
|
||||
|
||||
xh_e8m0 = torch.empty(
|
||||
padded_rows, padded_cols, dtype=torch.float8_e8m0fnu, device=a.device
|
||||
)
|
||||
|
||||
if not hasattr(torch.ops, "_qutlass_C"):
|
||||
raise RuntimeError(
|
||||
"The `_qutlass_C` extension is not loaded. "
|
||||
"Make sure your custom op library is imported before calling fusedQuantizeMx."
|
||||
)
|
||||
|
||||
if method == "quest":
|
||||
return torch.ops._qutlass_C.fusedQuantizeMxQuest(a, b, xh_e2m1, xh_e8m0)
|
||||
elif method == "abs_max":
|
||||
return torch.ops._qutlass_C.fusedQuantizeMxAbsMax(a, b, xh_e2m1, xh_e8m0)
|
||||
else:
|
||||
raise ValueError(f"invalid method {method!r}, must be 'quest' or 'abs_max'")
|
||||
|
||||
|
||||
if hasattr(torch.ops._qutlass_C, "fusedQuantizeNv"):
|
||||
|
||||
@register_fake("_qutlass_C::fusedQuantizeNv")
|
||||
def _fake_fused_quantize_nv(
|
||||
a: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
xh_e2m1: torch.Tensor,
|
||||
xh_e4m3: torch.Tensor,
|
||||
global_scale: torch.Tensor,
|
||||
):
|
||||
return xh_e2m1, xh_e4m3
|
||||
|
||||
|
||||
def fusedQuantizeNv(
|
||||
a: torch.Tensor, b: torch.Tensor, global_scale: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
xh_e2m1 = torch.empty(
|
||||
*a.shape[:-1], a.size(-1) // 2, dtype=torch.uint8, device=a.device
|
||||
)
|
||||
|
||||
rows, cols = a.numel() // a.size(-1), a.size(-1) // 16
|
||||
n_row_blocks = ceil_div(rows, 128)
|
||||
n_col_blocks = ceil_div(cols, 4)
|
||||
padded_rows = n_row_blocks * 128
|
||||
padded_cols = n_col_blocks * 4
|
||||
xh_e4m3 = torch.empty(
|
||||
padded_rows, padded_cols, dtype=torch.float8_e4m3fn, device=a.device
|
||||
)
|
||||
|
||||
return torch.ops._qutlass_C.fusedQuantizeNv(a, b, xh_e2m1, xh_e4m3, global_scale)
|
||||
|
||||
|
||||
def hadacore_transform(x: torch.Tensor, inplace: bool = True) -> torch.Tensor:
|
||||
"""
|
||||
Perform Hadamard transforms using [Hadacore](https://arxiv.org/abs/2412.08832)
|
||||
|
||||
Reference in New Issue
Block a user