[Bugfix][CPU] Fix several bugs (#54042)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
@@ -150,6 +150,15 @@ def _log_gdn_backend_decision(
|
||||
head_k_dim = getattr(
|
||||
vllm_config.model_config.hf_text_config, "linear_key_head_dim", None
|
||||
)
|
||||
|
||||
if current_platform.is_cpu():
|
||||
logger.info_once(
|
||||
"Using %s GDN prefill kernel (head_k_dim=%s).",
|
||||
"CPU",
|
||||
head_k_dim,
|
||||
)
|
||||
return
|
||||
|
||||
chosen = {
|
||||
"flashinfer": "FlashInfer",
|
||||
"cutedsl": "CuteDSL",
|
||||
@@ -492,7 +501,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
envs.VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE
|
||||
)
|
||||
self.gdn_decode_kernel = envs.VLLM_GDN_DECODE_KERNEL.strip().lower()
|
||||
if self.gdn_decode_kernel == "cuda":
|
||||
if self.gdn_decode_kernel == "cuda" and current_platform.is_cuda_alike():
|
||||
reason = self._fused_gdn_decode_unsupported_reason(vllm_config)
|
||||
if reason is not None:
|
||||
if "VLLM_GDN_DECODE_KERNEL" in os.environ:
|
||||
@@ -503,6 +512,9 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
"Falling back to the Triton GDN decode path: %s", reason
|
||||
)
|
||||
self.gdn_decode_kernel = "triton"
|
||||
elif current_platform.is_cpu():
|
||||
self.gdn_decode_kernel = "CPU"
|
||||
|
||||
self.enable_fused_gdn_decode = self.gdn_decode_kernel == "cuda"
|
||||
logger.info_once("GDN decode kernel: %s", self.gdn_decode_kernel)
|
||||
|
||||
|
||||
@@ -505,13 +505,15 @@ def dispatch_cpu_unquantized_gemm(
|
||||
# cross over to favoring oneDNN once batch size grows past decode-sized
|
||||
# M, so they keep using oneDNN below.
|
||||
if check_cpu_sgl_kernel(N, K, dtype):
|
||||
# For small size GEMM, packed_weight might be float32
|
||||
packed_weight = torch.ops._C.convert_weight_packed(layer.weight)
|
||||
if getattr(layer, "bias", None) is not None:
|
||||
bias_f32 = layer.bias.to(torch.float32)
|
||||
else:
|
||||
bias_f32 = None
|
||||
layer.cpu_linear = lambda x, weight, bias: torch.ops._C.weight_packed_linear(
|
||||
x, packed_weight, bias_f32 if bias is not None else None, True
|
||||
layer.bias.data = layer.bias.to(torch.float32)
|
||||
layer.cpu_linear = lambda x, weight, bias: ops.weight_packed_linear_cpu(
|
||||
x,
|
||||
packed_weight,
|
||||
N,
|
||||
bias,
|
||||
)
|
||||
if remove_weight:
|
||||
layer.weight = torch.nn.Parameter(torch.empty(0), requires_grad=False)
|
||||
|
||||
Reference in New Issue
Block a user