[CPU] Support int8 compute mode in CPU AWQ (#35697)
Signed-off-by: Yintong Lu <yintong.lu@intel.com>
This commit is contained in:
@@ -2967,6 +2967,38 @@ if hasattr(torch.ops._C, "int8_scaled_mm_with_quant"):
|
||||
return torch.empty((M, N), dtype=out_dtype)
|
||||
|
||||
|
||||
if hasattr(torch.ops._C, "convert_weight_packed_scale_zp"):
|
||||
|
||||
@register_fake("_C::convert_weight_packed_scale_zp")
|
||||
def convert_weight_packed_scale_zp_fake(
|
||||
qweight: torch.Tensor,
|
||||
qzeros: torch.Tensor,
|
||||
scales: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
return (
|
||||
torch.empty_like(qweight),
|
||||
torch.empty_like(qzeros),
|
||||
torch.empty_like(scales),
|
||||
)
|
||||
|
||||
|
||||
if hasattr(torch.ops._C, "int4_scaled_mm_cpu"):
|
||||
|
||||
@register_fake("_C::int4_scaled_mm_cpu")
|
||||
def int4_scaled_mm_cpu_fake(
|
||||
x: torch.Tensor,
|
||||
w: torch.Tensor,
|
||||
w_zeros: torch.Tensor,
|
||||
w_scales: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
N = w_scales.size(0) * w_scales.size(-1)
|
||||
return torch.empty((x.size(0), N), dtype=x.dtype, device=x.device)
|
||||
|
||||
|
||||
_supports_cpu_w4a8_int8 = bool(hasattr(torch.ops._C, "convert_weight_packed_scale_zp"))
|
||||
|
||||
|
||||
class CPUDNNLGEMMHandler:
|
||||
def __init__(self) -> None:
|
||||
self.handler_tensor: torch.Tensor | None = None
|
||||
|
||||
Reference in New Issue
Block a user