[CPU] Refactor CPU unquantized linear (#24150)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
@@ -1928,6 +1928,35 @@ class CPUDNNLGEMMHandler:
|
||||
torch.ops._C.release_dnnl_matmul_handler(self.handler)
|
||||
|
||||
|
||||
if hasattr(torch.ops._C, "create_onednn_mm_handler"):
|
||||
_supports_onednn = True
|
||||
else:
|
||||
_supports_onednn = False
|
||||
|
||||
|
||||
def create_onednn_mm(
|
||||
weight: torch.Tensor, # [K, N]
|
||||
primitive_cache_size: int = 128,
|
||||
) -> CPUDNNLGEMMHandler:
|
||||
handler = CPUDNNLGEMMHandler()
|
||||
handler.k, handler.n = weight.size()
|
||||
handler.handler = torch.ops._C.create_onednn_mm_handler(
|
||||
weight, primitive_cache_size)
|
||||
return handler
|
||||
|
||||
|
||||
def onednn_mm(
|
||||
dnnl_handler: CPUDNNLGEMMHandler,
|
||||
x: torch.Tensor,
|
||||
bias: Optional[torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
output = torch.empty((*x.shape[0:-1], dnnl_handler.n), dtype=x.dtype)
|
||||
torch.ops._C.onednn_mm(output, x.reshape(-1, dnnl_handler.k), bias,
|
||||
dnnl_handler.handler)
|
||||
|
||||
return output
|
||||
|
||||
|
||||
def create_onednn_scaled_mm(
|
||||
weight: torch.Tensor, # [K, N]
|
||||
weight_scales: torch.Tensor,
|
||||
|
||||
@@ -9,7 +9,6 @@ import torch
|
||||
import torch.nn as nn
|
||||
from torch.nn.parameter import Parameter, UninitializedParameter
|
||||
|
||||
from vllm import envs
|
||||
from vllm.distributed import (divide, get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
split_tensor_along_last_dim,
|
||||
@@ -200,26 +199,10 @@ class UnquantizedLinearMethod(LinearMethodBase):
|
||||
set_weight_attrs(weight, extra_weight_attrs)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
# special postprocessing for CPU SGL
|
||||
if current_platform.is_cpu() and envs.VLLM_CPU_SGL_KERNEL:
|
||||
from vllm.model_executor.layers.utils import check_cpu_sgl_kernel
|
||||
N, K = layer.weight.size()
|
||||
dtype = layer.weight.dtype
|
||||
if check_cpu_sgl_kernel(N, K, dtype):
|
||||
packed_weight = torch.ops._C.convert_weight_packed(
|
||||
layer.weight)
|
||||
assert packed_weight.size() == layer.weight.size()
|
||||
layer.weight.copy_(packed_weight)
|
||||
if layer.bias is not None:
|
||||
layer.bias = Parameter(layer.bias.to(torch.float32),
|
||||
requires_grad=False)
|
||||
layer.use_cpu_sgl = True
|
||||
else:
|
||||
logger.warning(
|
||||
"CPU SGL kernels require Intel AMX support,"
|
||||
" bf16/fp16/int8 weight, IC and OC are divisible by "
|
||||
"32 and 16.")
|
||||
layer.use_cpu_sgl = False
|
||||
if current_platform.is_cpu():
|
||||
from vllm.model_executor.layers.utils import (
|
||||
dispatch_cpu_unquantized_gemm)
|
||||
dispatch_cpu_unquantized_gemm(layer, remove_weight=True)
|
||||
|
||||
def apply(self,
|
||||
layer: torch.nn.Module,
|
||||
|
||||
@@ -142,20 +142,49 @@ direct_register_custom_op(
|
||||
)
|
||||
|
||||
|
||||
def check_cpu_sgl_kernel(n: int, k: int, dtype: torch.dtype):
|
||||
def check_cpu_sgl_kernel(n: int, k: int, dtype: torch.dtype) -> bool:
|
||||
return (torch._C._cpu._is_amx_tile_supported()
|
||||
and (dtype in (torch.bfloat16, torch.int8)) and k % 32 == 0
|
||||
and n % 16 == 0)
|
||||
|
||||
|
||||
def dispatch_cpu_unquantized_gemm(
|
||||
layer: torch.nn.Module,
|
||||
remove_weight: bool,
|
||||
) -> None:
|
||||
N, K = layer.weight.size()
|
||||
dtype = layer.weight.dtype
|
||||
if envs.VLLM_CPU_SGL_KERNEL and check_cpu_sgl_kernel(N, K, dtype):
|
||||
packed_weight = torch.ops._C.convert_weight_packed(layer.weight)
|
||||
if getattr(layer, "bias", None) is not None:
|
||||
bias_f32 = layer.bias.to(torch.float32)
|
||||
else:
|
||||
bias_f32 = None
|
||||
layer.cpu_linear = (
|
||||
lambda x, weight, bias: torch.ops._C.weight_packed_linear(
|
||||
x, packed_weight, bias_f32
|
||||
if bias is not None else None, True))
|
||||
if remove_weight:
|
||||
layer.weight = torch.nn.Parameter(torch.empty(0),
|
||||
requires_grad=False)
|
||||
elif ops._supports_onednn:
|
||||
origin_weight = layer.weight
|
||||
if remove_weight:
|
||||
layer.weight = torch.nn.Parameter(torch.empty(0),
|
||||
requires_grad=False)
|
||||
handler = ops.create_onednn_mm(origin_weight.t(), 32)
|
||||
layer.cpu_linear = lambda x, weight, bias: ops.onednn_mm(
|
||||
handler, x, bias)
|
||||
else:
|
||||
layer.cpu_linear = lambda x, weight, bias: torch.nn.functional.linear(
|
||||
x, weight, bias)
|
||||
|
||||
|
||||
def cpu_unquantized_gemm(layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
bias: Optional[torch.Tensor] = None):
|
||||
if getattr(layer, "use_cpu_sgl", False):
|
||||
return torch.ops._C.weight_packed_linear(x, weight, bias, True)
|
||||
else:
|
||||
return torch.nn.functional.linear(x, weight, bias)
|
||||
return layer.cpu_linear(x, weight, bias)
|
||||
|
||||
|
||||
def dispatch_unquantized_gemm() -> Callable[..., torch.Tensor]:
|
||||
|
||||
@@ -40,6 +40,12 @@ class UnquantizedEmbeddingMethod(QuantizeMethodBase):
|
||||
layer.register_parameter("weight", weight)
|
||||
set_weight_attrs(weight, extra_weight_attrs)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
if current_platform.is_cpu():
|
||||
from vllm.model_executor.layers.utils import (
|
||||
dispatch_cpu_unquantized_gemm)
|
||||
dispatch_cpu_unquantized_gemm(layer, remove_weight=False)
|
||||
|
||||
def apply(self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user