[Bugfix][CPU] Fix several bugs (#54042)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
+11
-12
@@ -3328,18 +3328,17 @@ def dsv3_fused_a_gemm(
|
||||
torch.ops._C.dsv3_fused_a_gemm(output, mat_a, mat_b, enable_pdl)
|
||||
|
||||
|
||||
if hasattr(torch.ops._C, "weight_packed_linear"):
|
||||
|
||||
@register_fake("_C::weight_packed_linear")
|
||||
def weight_packed_linear_fake(
|
||||
mat1: torch.Tensor,
|
||||
mat2: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
is_vnni: bool,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty(
|
||||
(mat1.size(0), mat2.size(0)), dtype=mat1.dtype, device=mat2.device
|
||||
)
|
||||
def weight_packed_linear_cpu(
|
||||
x: torch.Tensor,
|
||||
packed_weight: torch.Tensor,
|
||||
n: int,
|
||||
bias: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
output = torch.empty((*x.shape[0:-1], n), dtype=x.dtype)
|
||||
torch.ops._C.weight_packed_linear(
|
||||
output.reshape(-1, n), x.reshape(-1, x.size(-1)), packed_weight, bias, True
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
class CPUQuantMethod(IntEnum):
|
||||
|
||||
@@ -150,6 +150,15 @@ def _log_gdn_backend_decision(
|
||||
head_k_dim = getattr(
|
||||
vllm_config.model_config.hf_text_config, "linear_key_head_dim", None
|
||||
)
|
||||
|
||||
if current_platform.is_cpu():
|
||||
logger.info_once(
|
||||
"Using %s GDN prefill kernel (head_k_dim=%s).",
|
||||
"CPU",
|
||||
head_k_dim,
|
||||
)
|
||||
return
|
||||
|
||||
chosen = {
|
||||
"flashinfer": "FlashInfer",
|
||||
"cutedsl": "CuteDSL",
|
||||
@@ -492,7 +501,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
envs.VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE
|
||||
)
|
||||
self.gdn_decode_kernel = envs.VLLM_GDN_DECODE_KERNEL.strip().lower()
|
||||
if self.gdn_decode_kernel == "cuda":
|
||||
if self.gdn_decode_kernel == "cuda" and current_platform.is_cuda_alike():
|
||||
reason = self._fused_gdn_decode_unsupported_reason(vllm_config)
|
||||
if reason is not None:
|
||||
if "VLLM_GDN_DECODE_KERNEL" in os.environ:
|
||||
@@ -503,6 +512,9 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
"Falling back to the Triton GDN decode path: %s", reason
|
||||
)
|
||||
self.gdn_decode_kernel = "triton"
|
||||
elif current_platform.is_cpu():
|
||||
self.gdn_decode_kernel = "CPU"
|
||||
|
||||
self.enable_fused_gdn_decode = self.gdn_decode_kernel == "cuda"
|
||||
logger.info_once("GDN decode kernel: %s", self.gdn_decode_kernel)
|
||||
|
||||
|
||||
@@ -505,13 +505,15 @@ def dispatch_cpu_unquantized_gemm(
|
||||
# cross over to favoring oneDNN once batch size grows past decode-sized
|
||||
# M, so they keep using oneDNN below.
|
||||
if check_cpu_sgl_kernel(N, K, dtype):
|
||||
# For small size GEMM, packed_weight might be float32
|
||||
packed_weight = torch.ops._C.convert_weight_packed(layer.weight)
|
||||
if getattr(layer, "bias", None) is not None:
|
||||
bias_f32 = layer.bias.to(torch.float32)
|
||||
else:
|
||||
bias_f32 = None
|
||||
layer.cpu_linear = lambda x, weight, bias: torch.ops._C.weight_packed_linear(
|
||||
x, packed_weight, bias_f32 if bias is not None else None, True
|
||||
layer.bias.data = layer.bias.to(torch.float32)
|
||||
layer.cpu_linear = lambda x, weight, bias: ops.weight_packed_linear_cpu(
|
||||
x,
|
||||
packed_weight,
|
||||
N,
|
||||
bias,
|
||||
)
|
||||
if remove_weight:
|
||||
layer.weight = torch.nn.Parameter(torch.empty(0), requires_grad=False)
|
||||
|
||||
@@ -34,6 +34,20 @@ if HAS_TRITON:
|
||||
if name != "cpu" and x.driver and x.driver.is_active()
|
||||
]
|
||||
|
||||
# Check Triton CPU
|
||||
if "cpu" in version("vllm"):
|
||||
if "cpu" in backends:
|
||||
HAS_TRITON = True
|
||||
# Suppress following warnings on CPU-only platforms
|
||||
if len(active_drivers) == 0:
|
||||
active_drivers.append(backends["cpu"].driver)
|
||||
else:
|
||||
logger.warning(
|
||||
"Triton is installed, but doesn't include CPU backend. "
|
||||
"Disabling Triton."
|
||||
)
|
||||
HAS_TRITON = False
|
||||
|
||||
# Check if we're in a distributed environment where CUDA_VISIBLE_DEVICES
|
||||
# or HIP_VISIBLE_DEVICES might be temporarily empty (e.g., Ray sets it to ""
|
||||
# during actor init)
|
||||
@@ -63,17 +77,6 @@ if HAS_TRITON:
|
||||
len(active_drivers),
|
||||
)
|
||||
HAS_TRITON = False
|
||||
|
||||
# Check Triton CPU
|
||||
if "cpu" in version("vllm"):
|
||||
if "cpu" in backends:
|
||||
HAS_TRITON = True
|
||||
else:
|
||||
logger.warning(
|
||||
"Triton is installed, but doesn't include CPU backend. "
|
||||
"Disabling Triton."
|
||||
)
|
||||
HAS_TRITON = False
|
||||
except ImportError:
|
||||
# This can occur if Triton is partially installed or triton.backends
|
||||
# is missing.
|
||||
|
||||
@@ -13,7 +13,6 @@ from vllm import _custom_ops as ops
|
||||
from vllm import envs
|
||||
from vllm.config import (
|
||||
VllmConfig,
|
||||
get_current_vllm_config,
|
||||
get_layers_from_vllm_config,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
@@ -59,7 +58,7 @@ class CPUAttentionBackend(AttentionBackend):
|
||||
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes() -> list[int | MultipleOf]:
|
||||
return [MultipleOf(16)]
|
||||
return [MultipleOf(32)]
|
||||
|
||||
@classmethod
|
||||
def get_supported_head_sizes(cls) -> list[int]:
|
||||
@@ -152,7 +151,7 @@ class CPUAttentionMetadataBuilder(AttentionMetadataBuilder[CPUAttentionMetadata]
|
||||
self.head_dim = kv_cache_spec.head_size
|
||||
self.dtype = vllm_config.model_config.dtype
|
||||
self.window_size = self._group_sliding_window()
|
||||
self.block_size = vllm_config.cache_config.block_size
|
||||
self.block_size = kv_cache_spec.block_size
|
||||
self.kv_cache_dtype = vllm_config.cache_config.cache_dtype
|
||||
self.isa = _get_attn_isa(
|
||||
self.dtype,
|
||||
@@ -160,11 +159,21 @@ class CPUAttentionMetadataBuilder(AttentionMetadataBuilder[CPUAttentionMetadata]
|
||||
self.head_dim,
|
||||
self.kv_cache_dtype,
|
||||
)
|
||||
self._set_isa_to_layers(layer_names)
|
||||
self.is_cross_attention = isinstance(kv_cache_spec, CrossAttentionSpec)
|
||||
self.is_encoder_only_attention = isinstance(
|
||||
kv_cache_spec, EncoderOnlyAttentionSpec
|
||||
)
|
||||
|
||||
def _set_isa_to_layers(self, layer_names: list[str]) -> None:
|
||||
attn_layers = get_layers_from_vllm_config(
|
||||
self.vllm_config,
|
||||
Attention,
|
||||
layer_names,
|
||||
)
|
||||
for layer in attn_layers.values():
|
||||
layer.isa = self.isa # type: ignore
|
||||
|
||||
def _group_sliding_window(self) -> int:
|
||||
"""The window shared by every layer in this group, else -1 (no window).
|
||||
|
||||
@@ -325,14 +334,6 @@ class CPUAttentionBackendImpl(AttentionImpl):
|
||||
"heads in the layer"
|
||||
)
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
self.isa = _get_attn_isa(
|
||||
vllm_config.model_config.dtype,
|
||||
vllm_config.cache_config.block_size,
|
||||
self.head_size,
|
||||
self.kv_cache_dtype,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
layer: AttentionLayer,
|
||||
@@ -391,7 +392,7 @@ class CPUAttentionBackendImpl(AttentionImpl):
|
||||
key_cache,
|
||||
value_cache,
|
||||
attn_metadata.slot_mapping,
|
||||
self.isa,
|
||||
layer.isa, # type: ignore
|
||||
k_scale=layer._k_scale_float,
|
||||
v_scale=layer._v_scale_float,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
@@ -440,7 +441,7 @@ class CPUAttentionBackendImpl(AttentionImpl):
|
||||
key_cache,
|
||||
value_cache,
|
||||
slot_mapping,
|
||||
self.isa,
|
||||
layer.isa, # type: ignore
|
||||
k_scale=layer._k_scale_float,
|
||||
v_scale=layer._v_scale_float,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
|
||||
Reference in New Issue
Block a user