[Hardware][CPU] Enable granite-4 model on cpu (#47641)

Signed-off-by: Akash Kaothalkar <akashkaothalkar@akashs-mbp.bl1-in.ibm.com>
Signed-off-by: Akash Kaothalkar <akashkaothalkar@dhcp-9-123-5-76.bl1-in.ibm.com>
Signed-off-by: Akash Kaothalkar <akashkaothalkar@Akashs-MBP.lan>
Signed-off-by: Akash kaothalkar <akash.kaothalkar@ibm.com>
Co-authored-by: Akash Kaothalkar <akashkaothalkar@dhcp-9-123-5-76.bl1-in.ibm.com>
Co-authored-by: Akash Kaothalkar <akashkaothalkar@Akashs-MBP.lan>
Co-authored-by: Akash Kaothalkar <akashkaothalkar@akashs-mbp.bl1-in.ibm.com>
Co-authored-by: Akash kaothalkar <akash.kaothalkar@ibm.com>
Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
Akash kaothalkar
2026-07-20 06:15:16 +00:00
committed by GitHub
co-authored by Akash Kaothalkar Akash Kaothalkar Akash Kaothalkar Akash kaothalkar Li, Jiang <jiang1.li@intel.com>
parent dcfebf93f4
commit 5c9f6557d7
21 changed files with 1283 additions and 62 deletions
@@ -1237,3 +1237,15 @@ def causal_conv1d_update(
if unsqueeze:
out = out.squeeze(-1)
return out.to(original_x_dtype)
from vllm.platforms import current_platform # noqa: E402
if current_platform.is_cpu():
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
causal_conv1d_fn_cpu,
causal_conv1d_update_cpu,
)
causal_conv1d_fn = causal_conv1d_fn_cpu # type: ignore
causal_conv1d_update = causal_conv1d_update_cpu # type: ignore
@@ -6,18 +6,31 @@ from __future__ import annotations
import torch
import torch.nn.functional as F
from vllm._custom_ops import causal_conv1d_update_cpu_vec
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID, PAD_SLOT_ID
# for prefill
def causal_conv1d_torch(
def causal_conv1d_fn_cpu(
x: torch.Tensor,
weight: torch.Tensor,
bias: torch.Tensor | None,
conv_states: torch.Tensor,
query_start_loc: torch.Tensor,
cache_indices: torch.Tensor,
has_initial_state: torch.Tensor,
cache_indices: torch.Tensor | None = None,
has_initial_state: torch.Tensor | None = None,
activation: str | None = "silu",
pad_slot_id: int = PAD_SLOT_ID,
**kwargs,
) -> torch.Tensor:
"""CPU implementation for causal_conv1d_fwd."""
if isinstance(activation, bool) and activation:
activation = "silu"
elif isinstance(activation, bool):
activation = None
original_x_dtype = x.dtype
x = x.to(conv_states.dtype)
out = torch.empty_like(x)
state_len = weight.shape[1] - 1
assert activation in {None, "silu", "swish"}
@@ -27,11 +40,21 @@ def causal_conv1d_torch(
for idx in range(query_start_loc.shape[0] - 1)
]
weight = weight.unsqueeze(1)
for seq_idx, (bos, eos) in enumerate(seq_begin_end_idx):
slot = int(cache_indices[seq_idx].item())
if bos == eos:
continue
slot = (
int(cache_indices[seq_idx].item()) if cache_indices is not None else seq_idx
)
if slot == pad_slot_id:
continue
seq_x = x[:, bos:eos].unsqueeze(0)
if bool(has_initial_state[seq_idx].item()):
if has_initial_state is not None and bool(has_initial_state[seq_idx].item()):
initial_state = conv_states[slot, :, :state_len].unsqueeze(0)
else:
initial_state = torch.zeros(
@@ -51,16 +74,48 @@ def causal_conv1d_torch(
groups=weight.shape[0],
)
seq_out = seq_out[..., -seq_x.shape[-1] :].to(dtype=x.dtype)
if activation in ("silu", "swish"):
seq_out = F.silu(seq_out)
out[:, bos:eos] = seq_out.squeeze(0)
conv_states[slot, :, :state_len].copy_(conv_input[..., -state_len:].squeeze(0))
return out
return out.to(original_x_dtype)
def causal_conv1d_update_cpu(
x: torch.Tensor,
conv_state: torch.Tensor,
weight: torch.Tensor,
bias: torch.Tensor | None = None,
activation: bool | str | None = None,
conv_state_indices: torch.Tensor | None = None,
query_start_loc: torch.Tensor | None = None,
pad_slot_id: int | None = None,
**kwargs,
) -> torch.Tensor:
"""CPU implementation for causal_conv1d_update."""
if isinstance(activation, bool):
activation = "silu" if activation else None
if pad_slot_id is None:
pad_slot_id = kwargs.get("null_block_id", NULL_BLOCK_ID)
if pad_slot_id is None:
pad_slot_id = NULL_BLOCK_ID
return causal_conv1d_update_cpu_vec(
x,
conv_state,
weight,
bias,
activation,
conv_state_indices,
query_start_loc,
pad_slot_id,
)
# for decode
def causal_conv1d_update_torch(
x: torch.Tensor,
conv_state: torch.Tensor,
@@ -68,6 +123,11 @@ def causal_conv1d_update_torch(
bias: torch.Tensor | None = None,
activation: str | None = None,
) -> torch.Tensor:
"""
Pure PyTorch fallback for causal_conv1d_update.
Currently used as a fallback for Arm (aarch64) to leverage
oneDNN/ACL F.conv1d kernels for batched decoding.
"""
assert activation in {None, "silu", "swish"}
_, dim, seq_len = x.shape
@@ -10,9 +10,13 @@ import vllm._custom_ops as ops
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
causal_conv1d_torch,
causal_conv1d_fn_cpu as causal_conv1d_torch,
)
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
causal_conv1d_update_cpu,
causal_conv1d_update_torch,
)
from vllm.platforms import CpuArchEnum, current_platform
from vllm.utils.torch_utils import (
LayerNameType,
_resolve_layer_name,
@@ -140,21 +144,30 @@ def _cpu_gdn_attention_nonspec(
conv_states=conv_state,
weight=layer.conv1d.weight,
bias=layer.conv1d.bias,
silu_activation=layer.activation == "silu",
silu_activation=(layer.activation == "silu"),
conv_state_indices=decode_state_indices,
is_vnni=True,
)
else:
decode_conv_state = conv_state[decode_state_indices].contiguous()
decode_mixed_qkv = causal_conv1d_update_torch(
# [B, dim] -> [B, dim, 1]
x=decode_mixed_qkv.unsqueeze(-1),
conv_state=decode_conv_state,
weight=conv_weights,
bias=layer.conv1d.bias,
activation=layer.activation,
).squeeze(-1)
conv_state[decode_state_indices] = decode_conv_state
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
decode_conv_state = conv_state[decode_state_indices].contiguous()
decode_mixed_qkv = causal_conv1d_update_torch(
x=decode_mixed_qkv.unsqueeze(-1),
conv_state=decode_conv_state,
weight=conv_weights,
bias=layer.conv1d.bias,
activation=layer.activation,
).squeeze(-1)
conv_state[decode_state_indices] = decode_conv_state
else:
decode_mixed_qkv = causal_conv1d_update_cpu(
x=decode_mixed_qkv,
conv_state=conv_state,
weight=conv_weights,
bias=layer.conv1d.bias,
activation=layer.activation,
conv_state_indices=decode_state_indices,
)
query, key, value = layer.rearrange_mixed_qkv(decode_mixed_qkv)
@@ -495,17 +508,26 @@ def _spec_aware_nonspec(
decode_a = a[:num_decode_tokens]
decode_state_indices = state_indices_tensor[:num_decodes]
# Only the first ``width-1`` columns hold the real conv state.
decode_conv_state = conv_buf[decode_state_indices][
:, :, : width - 1
].contiguous()
decode_mixed_qkv = causal_conv1d_update_torch(
x=decode_mixed_qkv.unsqueeze(-1),
conv_state=decode_conv_state,
weight=conv_weights,
bias=layer.conv1d.bias,
activation=layer.activation,
).squeeze(-1)
conv_buf[decode_state_indices, :, : width - 1] = decode_conv_state
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
conv_state_view = conv_buf[:, :, : width - 1]
decode_conv_state = conv_state_view[decode_state_indices].contiguous()
decode_mixed_qkv = causal_conv1d_update_torch(
x=decode_mixed_qkv.unsqueeze(-1),
conv_state=decode_conv_state,
weight=conv_weights,
bias=layer.conv1d.bias,
activation=layer.activation,
).squeeze(-1)
conv_state_view[decode_state_indices] = decode_conv_state
else:
decode_mixed_qkv = causal_conv1d_update_cpu(
x=decode_mixed_qkv,
conv_state=conv_buf[:, :, : width - 1],
weight=conv_weights,
bias=layer.conv1d.bias,
activation=layer.activation,
conv_state_indices=decode_state_indices,
)
query, key, value = layer.rearrange_mixed_qkv(decode_mixed_qkv)
# rearrange_mixed_qkv can return views whose last dim is not
@@ -0,0 +1,144 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import vllm._custom_ops as ops
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
def _mamba_chunk_scan_combined_fwd_cpu(
x,
dt,
A,
B,
C,
chunk_size,
out,
D=None,
z=None,
dt_bias=None,
initial_states=None,
return_intermediate_states=False,
seq_idx=None,
cu_seqlens=None,
cu_chunk_seqlens=None,
last_chunk_indices=None,
dt_softplus=False,
dt_limit=(0.0, float("inf")),
state_dtype=None,
**kwargs,
):
seqlen, nheads, headdim = x.shape
_, ngroups, dstate = B.shape
assert cu_seqlens is not None
batch = cu_seqlens.size(0) - 1
dt_f = dt.float()
if dt_bias is not None:
dt_f = dt_f + dt_bias.float().unsqueeze(0)
if dt_softplus:
dt_f = torch.nn.functional.softplus(dt_f)
if dt_limit[0] > 0.0 or dt_limit[1] < float("inf"):
dt_f = dt_f.clamp(min=dt_limit[0], max=dt_limit[1])
all_states = torch.zeros(
batch, nheads, headdim, dstate, dtype=torch.float32, device=x.device
)
if initial_states is not None:
all_states.copy_(initial_states.float())
assert out.is_contiguous(), (
"_mamba_chunk_scan_combined_fwd_cpu: `out` must be "
"pre-allocated as a contiguous tensor"
)
D_1d = None
if D is not None:
d = D.float()
while d.dim() > 1 and d.stride(-1) == 0:
d = d.squeeze(-1)
D_1d = d.contiguous()
ops.mamba_chunk_scan_fwd_cpu(
out,
all_states,
x,
dt_f,
A,
B,
C,
D_1d,
z,
cu_seqlens.to(torch.int32),
)
out_dtype = state_dtype if state_dtype is not None else x.dtype
all_states = all_states.to(out_dtype)
return all_states
def selective_state_update(
state,
x,
dt,
A,
B,
C,
D=None,
z=None,
dt_bias=None,
dt_softplus=False,
state_batch_indices=None,
dst_state_batch_indices=None,
null_block_id=NULL_BLOCK_ID,
out=None,
num_accepted_tokens=None,
cu_seqlens=None,
is_blackwell=False,
enable_stochastic_rounding=False,
cache_philox_rounds=0,
):
"""CPU implementation for selective_state_update."""
# Ensure out tensor exists
if out is None:
out = torch.empty_like(x if x.dim() == 2 else x)
_state = state.unsqueeze(1) if state.dim() == 3 else state
_x = x.unsqueeze(1) if x.dim() == 2 else x
_dt = dt.unsqueeze(1) if dt.dim() == 2 else dt
_A = A.unsqueeze(0) if A.dim() == 2 else A
_B = B.unsqueeze(1) if B.dim() == 2 else B
_C = C.unsqueeze(1) if C.dim() == 2 else C
_D = D.unsqueeze(0) if (D is not None and D.dim() == 1) else D
_z = z.unsqueeze(1) if (z is not None and z.dim() == 2) else z
_dt_bias = (
dt_bias.unsqueeze(0)
if (dt_bias is not None and dt_bias.dim() == 1)
else dt_bias
)
_out = out.unsqueeze(1) if out.dim() == 2 else out
_sbi = state_batch_indices
_dsbi = dst_state_batch_indices
ops.selective_state_update_cpu(
_state,
_x,
_dt,
_A,
_B,
_C,
_D,
_z,
_dt_bias,
dt_softplus,
_sbi,
_dsbi,
null_block_id,
_out,
num_accepted_tokens,
cu_seqlens,
)
return _out.squeeze(1) if out.dim() == 2 else _out
@@ -845,3 +845,13 @@ def selective_scan_fn(
return delta # output written inplace to delta
else:
return z # output written inplace to z
from vllm.platforms import current_platform # noqa: E402
if current_platform.is_cpu():
from vllm.model_executor.layers.mamba.ops.cpu.mamba_ssm import (
selective_state_update as selective_state_update_cpu,
)
selective_state_update = selective_state_update_cpu # type: ignore
@@ -225,3 +225,11 @@ def mamba_chunk_scan_combined_varlen(
)
return varlen_states
from vllm.platforms import current_platform # noqa: E402
if current_platform.is_cpu():
import vllm.model_executor.layers.mamba.ops.cpu.mamba_ssm as cpu_mamba_ssm
_mamba_chunk_scan_combined_fwd = cpu_mamba_ssm._mamba_chunk_scan_combined_fwd_cpu # type: ignore
@@ -4,8 +4,9 @@
Dispatch module for Mamba selective state update (SSU) backends.
Provides a unified `selective_state_update` function that dispatches to
either the Triton or FlashInfer backend based on the configured
`MambaBackendEnum`. Follows SGLang's dispatch pattern adapted for vLLM.
the Triton, FlashInfer, or CPU backend based on the configured
`MambaBackendEnum`. On CPU-only platforms (PowerPC, x86 without CUDA)
the backend defaults to 'cpu'.
"""
from abc import ABC, abstractmethod
@@ -182,9 +183,75 @@ class FlashInferSSUBackend(MambaSSUBackend):
)
class CPUSSUBackend(MambaSSUBackend):
"""CPU SSU backend using the compiled C++ VSX/scalar kernel.
On CPU-only platforms (PowerPC, x86 without CUDA) this dispatches to
the vectorized C++ kernel registered as ``torch.ops._C.selective_state_update_cpu``.
That kernel uses vec_op SIMD intrinsics (VSX on ppc64le, AVX2 on x86,
scalar fallback elsewhere) and is parallelised with OpenMP across heads.
Falls back to the pure-PyTorch implementation only if the C++ op is
unavailable (e.g. a CPU-less build).
"""
def __init__(self, mamba_config: MambaConfig):
super().__init__(mamba_config)
from vllm import _custom_ops as ops
self._cpp_kernel = ops.selective_state_update_cpu
logger.info("CPUSSUBackend: using compiled C++ selective_state_update kernel.")
@property
def name(self) -> str:
return "cpu"
def __call__(
self,
state: torch.Tensor,
x: torch.Tensor,
dt: torch.Tensor,
A: torch.Tensor,
B: torch.Tensor,
C: torch.Tensor,
D: torch.Tensor,
dt_bias: torch.Tensor,
z: torch.Tensor | None = None,
dt_softplus: bool = False,
state_batch_indices: torch.Tensor | None = None,
dst_state_batch_indices: torch.Tensor | None = None,
null_block_id: int = NULL_BLOCK_ID,
out: torch.Tensor | None = None,
num_accepted_tokens: torch.Tensor | None = None,
cu_seqlens: torch.Tensor | None = None,
is_blackwell: bool = False,
) -> None:
# C++ kernel: state shape expected as (nstates, nheads, dim, dstate)
# The kernel writes in-place into `out` and updates `state`.
self._cpp_kernel(
state,
x,
dt,
A,
B,
C,
D,
z,
dt_bias,
dt_softplus,
state_batch_indices,
dst_state_batch_indices,
null_block_id,
out,
num_accepted_tokens,
cu_seqlens,
)
_BACKEND_REGISTRY: dict[MambaBackendEnum, type[MambaSSUBackend]] = {
MambaBackendEnum.TRITON: TritonSSUBackend,
MambaBackendEnum.FLASHINFER: FlashInferSSUBackend,
MambaBackendEnum.CPU: CPUSSUBackend,
}
_mamba_ssu_backend: MambaSSUBackend | None = None
@@ -210,6 +277,20 @@ def initialize_mamba_ssu_backend(
global _mamba_ssu_backend
backend = mamba_config.backend
# On CPU-only platforms (PowerPC, x86 without CUDA) Triton JIT is
# unstable or unavailable. Silently fall back to the CPU
# backend unless the user explicitly chose something other than "triton".
if backend == MambaBackendEnum.TRITON:
from vllm.platforms import current_platform
if current_platform.is_cpu():
logger.info(
"CPU platform detected: overriding Mamba SSU backend "
"from 'triton' to 'cpu'."
)
backend = MambaBackendEnum.CPU
if backend not in _BACKEND_REGISTRY:
raise ValueError(
f"Unknown Mamba SSU backend: {backend}. "
+25 -12
View File
@@ -94,9 +94,13 @@ class ShortConv(MambaBase, CustomOp):
# Reference torch causal conv1d; runs on all CPU platforms. AMX kernels
# for causal conv can be plugged in here later.
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
causal_conv1d_torch,
causal_conv1d_fn_cpu as causal_conv1d_torch,
)
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
causal_conv1d_update_cpu,
causal_conv1d_update_torch,
)
from vllm.platforms import CpuArchEnum, current_platform
forward_context = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata
@@ -164,17 +168,26 @@ class ShortConv(MambaBase, CustomOp):
if has_decode:
assert attn_metadata.state_indices_tensor_d is not None
state_indices_d = attn_metadata.state_indices_tensor_d.flatten()
Bx_d = (B_d * x_d).unsqueeze(-1) # (num_decodes, dim, 1)
# Advanced indexing returns a copy; update in-place then scatter back
gathered = conv_state[state_indices_d] # (num_decodes, dim, state_len)
out_d = causal_conv1d_update_torch(
Bx_d,
gathered,
conv_weights,
self.conv.bias,
activation=None,
).squeeze(-1) # (num_decodes, dim)
conv_state[state_indices_d] = gathered
Bx_d = B_d * x_d # (num_decodes, dim)
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
conv_state_view = conv_state[state_indices_d].contiguous()
out_d = causal_conv1d_update_torch(
Bx_d.unsqueeze(-1),
conv_state_view,
conv_weights,
self.conv.bias,
activation=None,
).squeeze(-1)
conv_state[state_indices_d] = conv_state_view
else:
out_d = causal_conv1d_update_cpu(
Bx_d,
conv_state,
conv_weights,
self.conv.bias,
activation=None,
conv_state_indices=state_indices_d,
)
conv_output_list.insert(0, C_d * out_d)
hidden_states_out = torch.vstack(conv_output_list)
+6 -2
View File
@@ -234,10 +234,14 @@ def dispatch_cpu_unquantized_gemm(
layer.cpu_linear = torch.nn.functional.linear
return
# Skip CPU GEMM dispatch for non-2D weights (e.g. MoE 3D expert weights).
# These layers are handled by their own specialized methods.
if layer.weight.ndim != 2:
# this is not a linear layer
# For now it should be a causal_conv1d op
if torch.cpu._is_amx_tile_supported():
# For now it should be a causal_conv1d op or MoE 3D expert weights
if torch.cpu._is_amx_tile_supported() and hasattr(
ops, "causal_conv1d_weight_pack"
):
# prepack conv weight
unpacked = (
layer.weight.view(