[Feat] Enable CompressedTensorW4A8Int for XPU (#37207)
Signed-off-by: Li, Tianmu <tianmu.li@intel.com>
This commit is contained in:
@@ -37,6 +37,26 @@ if hasattr(torch.ops._xpu_C, "fp8_gemm_w8a16"):
|
||||
return torch.empty((M, N), dtype=input.dtype, device=input.device)
|
||||
|
||||
|
||||
if hasattr(torch.ops._xpu_C, "int4_gemm_w4a8"):
|
||||
|
||||
@register_fake("_xpu_C::int4_gemm_w4a8")
|
||||
def _int4_gemm_w4a8_fake(
|
||||
input: torch.Tensor,
|
||||
input_scales: torch.Tensor,
|
||||
input_zero_points: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
weight_zp: torch.Tensor,
|
||||
group_size: int,
|
||||
g_idx: torch.Tensor | None = None,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
input_2d = input.view(-1, input.shape[-1])
|
||||
M = input_2d.size(0)
|
||||
N = q_weight.size(1)
|
||||
return torch.empty((M, N), dtype=torch.float16, device=input.device)
|
||||
|
||||
|
||||
if hasattr(torch.ops._xpu_C, "int4_gemm_w4a16"):
|
||||
|
||||
@register_fake("_xpu_C::int4_gemm_w4a16")
|
||||
@@ -87,6 +107,40 @@ _OPS_REGISTERED = False
|
||||
|
||||
|
||||
class xpu_ops:
|
||||
@staticmethod
|
||||
@torch.compile
|
||||
def dynamic_per_token_int8_quant_ref(
|
||||
input: torch.Tensor, use_sym_quant: bool, bits: int
|
||||
):
|
||||
original_sizes = input.size()
|
||||
# view is not safe in torch.compile if input is not contiguous
|
||||
input = input.reshape(
|
||||
-1, original_sizes[-1]
|
||||
) # Flatten except for the last dimension
|
||||
qmin = -(2 ** (bits - 1)) if use_sym_quant else 0
|
||||
qmax = 2 ** (bits - 1) - 1 if use_sym_quant else 2**bits - 1
|
||||
min_val = torch.min(input, dim=-1)[0].to(dtype=torch.float32).unsqueeze(-1)
|
||||
max_val = torch.max(input, dim=-1)[0].to(dtype=torch.float32).unsqueeze(-1)
|
||||
if use_sym_quant:
|
||||
scale = (
|
||||
torch.maximum(torch.abs(min_val), torch.abs(max_val)) / qmax
|
||||
).clamp(min=1e-5)
|
||||
zero_point = torch.zeros_like(scale).to(dtype=torch.int32)
|
||||
else:
|
||||
scale = ((max_val - min_val) / qmax).clamp(min=1e-5)
|
||||
zero_point = -1 * torch.round(min_val / scale).to(dtype=torch.int32)
|
||||
scale = scale.to(dtype=input.dtype)
|
||||
quantized = torch.clamp(
|
||||
torch.round(input / scale.to(dtype=torch.float32) + zero_point),
|
||||
qmin,
|
||||
qmax,
|
||||
).to(dtype=torch.int8 if use_sym_quant else torch.uint8)
|
||||
return (
|
||||
quantized.view(original_sizes),
|
||||
scale.view(original_sizes[:-1] + (1,)),
|
||||
zero_point.view(original_sizes[:-1] + (1,)),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def flash_attn_varlen_func(
|
||||
q: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user