[Feat] Enable CompressedTensorW4A8Int for XPU (#37207)

Signed-off-by: Li, Tianmu <tianmu.li@intel.com>
This commit is contained in:
Tianmu Li
2026-03-20 02:34:28 +00:00
committed by GitHub
parent 269bf46d99
commit 47b7af0d87
4 changed files with 172 additions and 0 deletions
+54
View File
@@ -37,6 +37,26 @@ if hasattr(torch.ops._xpu_C, "fp8_gemm_w8a16"):
return torch.empty((M, N), dtype=input.dtype, device=input.device)
if hasattr(torch.ops._xpu_C, "int4_gemm_w4a8"):
@register_fake("_xpu_C::int4_gemm_w4a8")
def _int4_gemm_w4a8_fake(
input: torch.Tensor,
input_scales: torch.Tensor,
input_zero_points: torch.Tensor,
q_weight: torch.Tensor,
weight_scale: torch.Tensor,
weight_zp: torch.Tensor,
group_size: int,
g_idx: torch.Tensor | None = None,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
input_2d = input.view(-1, input.shape[-1])
M = input_2d.size(0)
N = q_weight.size(1)
return torch.empty((M, N), dtype=torch.float16, device=input.device)
if hasattr(torch.ops._xpu_C, "int4_gemm_w4a16"):
@register_fake("_xpu_C::int4_gemm_w4a16")
@@ -87,6 +107,40 @@ _OPS_REGISTERED = False
class xpu_ops:
@staticmethod
@torch.compile
def dynamic_per_token_int8_quant_ref(
input: torch.Tensor, use_sym_quant: bool, bits: int
):
original_sizes = input.size()
# view is not safe in torch.compile if input is not contiguous
input = input.reshape(
-1, original_sizes[-1]
) # Flatten except for the last dimension
qmin = -(2 ** (bits - 1)) if use_sym_quant else 0
qmax = 2 ** (bits - 1) - 1 if use_sym_quant else 2**bits - 1
min_val = torch.min(input, dim=-1)[0].to(dtype=torch.float32).unsqueeze(-1)
max_val = torch.max(input, dim=-1)[0].to(dtype=torch.float32).unsqueeze(-1)
if use_sym_quant:
scale = (
torch.maximum(torch.abs(min_val), torch.abs(max_val)) / qmax
).clamp(min=1e-5)
zero_point = torch.zeros_like(scale).to(dtype=torch.int32)
else:
scale = ((max_val - min_val) / qmax).clamp(min=1e-5)
zero_point = -1 * torch.round(min_val / scale).to(dtype=torch.int32)
scale = scale.to(dtype=input.dtype)
quantized = torch.clamp(
torch.round(input / scale.to(dtype=torch.float32) + zero_point),
qmin,
qmax,
).to(dtype=torch.int8 if use_sym_quant else torch.uint8)
return (
quantized.view(original_sizes),
scale.view(original_sizes[:-1] + (1,)),
zero_point.view(original_sizes[:-1] + (1,)),
)
@staticmethod
def flash_attn_varlen_func(
q: torch.Tensor,