[Hardware][CPU] Enable granite-4 model on cpu (#47641)
Signed-off-by: Akash Kaothalkar <akashkaothalkar@akashs-mbp.bl1-in.ibm.com> Signed-off-by: Akash Kaothalkar <akashkaothalkar@dhcp-9-123-5-76.bl1-in.ibm.com> Signed-off-by: Akash Kaothalkar <akashkaothalkar@Akashs-MBP.lan> Signed-off-by: Akash kaothalkar <akash.kaothalkar@ibm.com> Co-authored-by: Akash Kaothalkar <akashkaothalkar@dhcp-9-123-5-76.bl1-in.ibm.com> Co-authored-by: Akash Kaothalkar <akashkaothalkar@Akashs-MBP.lan> Co-authored-by: Akash Kaothalkar <akashkaothalkar@akashs-mbp.bl1-in.ibm.com> Co-authored-by: Akash kaothalkar <akash.kaothalkar@ibm.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
co-authored by
Akash Kaothalkar
Akash Kaothalkar
Akash Kaothalkar
Akash kaothalkar
Li, Jiang <jiang1.li@intel.com>
parent
dcfebf93f4
commit
5c9f6557d7
@@ -1237,3 +1237,15 @@ def causal_conv1d_update(
|
||||
if unsqueeze:
|
||||
out = out.squeeze(-1)
|
||||
return out.to(original_x_dtype)
|
||||
|
||||
|
||||
from vllm.platforms import current_platform # noqa: E402
|
||||
|
||||
if current_platform.is_cpu():
|
||||
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
|
||||
causal_conv1d_fn_cpu,
|
||||
causal_conv1d_update_cpu,
|
||||
)
|
||||
|
||||
causal_conv1d_fn = causal_conv1d_fn_cpu # type: ignore
|
||||
causal_conv1d_update = causal_conv1d_update_cpu # type: ignore
|
||||
|
||||
@@ -6,18 +6,31 @@ from __future__ import annotations
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm._custom_ops import causal_conv1d_update_cpu_vec
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID, PAD_SLOT_ID
|
||||
|
||||
# for prefill
|
||||
def causal_conv1d_torch(
|
||||
|
||||
def causal_conv1d_fn_cpu(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
conv_states: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
cache_indices: torch.Tensor,
|
||||
has_initial_state: torch.Tensor,
|
||||
cache_indices: torch.Tensor | None = None,
|
||||
has_initial_state: torch.Tensor | None = None,
|
||||
activation: str | None = "silu",
|
||||
pad_slot_id: int = PAD_SLOT_ID,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
"""CPU implementation for causal_conv1d_fwd."""
|
||||
if isinstance(activation, bool) and activation:
|
||||
activation = "silu"
|
||||
elif isinstance(activation, bool):
|
||||
activation = None
|
||||
|
||||
original_x_dtype = x.dtype
|
||||
x = x.to(conv_states.dtype)
|
||||
|
||||
out = torch.empty_like(x)
|
||||
state_len = weight.shape[1] - 1
|
||||
assert activation in {None, "silu", "swish"}
|
||||
@@ -27,11 +40,21 @@ def causal_conv1d_torch(
|
||||
for idx in range(query_start_loc.shape[0] - 1)
|
||||
]
|
||||
weight = weight.unsqueeze(1)
|
||||
|
||||
for seq_idx, (bos, eos) in enumerate(seq_begin_end_idx):
|
||||
slot = int(cache_indices[seq_idx].item())
|
||||
if bos == eos:
|
||||
continue
|
||||
|
||||
slot = (
|
||||
int(cache_indices[seq_idx].item()) if cache_indices is not None else seq_idx
|
||||
)
|
||||
|
||||
if slot == pad_slot_id:
|
||||
continue
|
||||
|
||||
seq_x = x[:, bos:eos].unsqueeze(0)
|
||||
if bool(has_initial_state[seq_idx].item()):
|
||||
|
||||
if has_initial_state is not None and bool(has_initial_state[seq_idx].item()):
|
||||
initial_state = conv_states[slot, :, :state_len].unsqueeze(0)
|
||||
else:
|
||||
initial_state = torch.zeros(
|
||||
@@ -51,16 +74,48 @@ def causal_conv1d_torch(
|
||||
groups=weight.shape[0],
|
||||
)
|
||||
seq_out = seq_out[..., -seq_x.shape[-1] :].to(dtype=x.dtype)
|
||||
|
||||
if activation in ("silu", "swish"):
|
||||
seq_out = F.silu(seq_out)
|
||||
|
||||
out[:, bos:eos] = seq_out.squeeze(0)
|
||||
conv_states[slot, :, :state_len].copy_(conv_input[..., -state_len:].squeeze(0))
|
||||
|
||||
return out
|
||||
return out.to(original_x_dtype)
|
||||
|
||||
|
||||
def causal_conv1d_update_cpu(
|
||||
x: torch.Tensor,
|
||||
conv_state: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
activation: bool | str | None = None,
|
||||
conv_state_indices: torch.Tensor | None = None,
|
||||
query_start_loc: torch.Tensor | None = None,
|
||||
pad_slot_id: int | None = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
"""CPU implementation for causal_conv1d_update."""
|
||||
if isinstance(activation, bool):
|
||||
activation = "silu" if activation else None
|
||||
|
||||
if pad_slot_id is None:
|
||||
pad_slot_id = kwargs.get("null_block_id", NULL_BLOCK_ID)
|
||||
if pad_slot_id is None:
|
||||
pad_slot_id = NULL_BLOCK_ID
|
||||
|
||||
return causal_conv1d_update_cpu_vec(
|
||||
x,
|
||||
conv_state,
|
||||
weight,
|
||||
bias,
|
||||
activation,
|
||||
conv_state_indices,
|
||||
query_start_loc,
|
||||
pad_slot_id,
|
||||
)
|
||||
|
||||
|
||||
# for decode
|
||||
def causal_conv1d_update_torch(
|
||||
x: torch.Tensor,
|
||||
conv_state: torch.Tensor,
|
||||
@@ -68,6 +123,11 @@ def causal_conv1d_update_torch(
|
||||
bias: torch.Tensor | None = None,
|
||||
activation: str | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Pure PyTorch fallback for causal_conv1d_update.
|
||||
Currently used as a fallback for Arm (aarch64) to leverage
|
||||
oneDNN/ACL F.conv1d kernels for batched decoding.
|
||||
"""
|
||||
assert activation in {None, "silu", "swish"}
|
||||
|
||||
_, dim, seq_len = x.shape
|
||||
|
||||
@@ -10,9 +10,13 @@ import vllm._custom_ops as ops
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
|
||||
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
|
||||
causal_conv1d_torch,
|
||||
causal_conv1d_fn_cpu as causal_conv1d_torch,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
|
||||
causal_conv1d_update_cpu,
|
||||
causal_conv1d_update_torch,
|
||||
)
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_resolve_layer_name,
|
||||
@@ -140,21 +144,30 @@ def _cpu_gdn_attention_nonspec(
|
||||
conv_states=conv_state,
|
||||
weight=layer.conv1d.weight,
|
||||
bias=layer.conv1d.bias,
|
||||
silu_activation=layer.activation == "silu",
|
||||
silu_activation=(layer.activation == "silu"),
|
||||
conv_state_indices=decode_state_indices,
|
||||
is_vnni=True,
|
||||
)
|
||||
else:
|
||||
decode_conv_state = conv_state[decode_state_indices].contiguous()
|
||||
decode_mixed_qkv = causal_conv1d_update_torch(
|
||||
# [B, dim] -> [B, dim, 1]
|
||||
x=decode_mixed_qkv.unsqueeze(-1),
|
||||
conv_state=decode_conv_state,
|
||||
weight=conv_weights,
|
||||
bias=layer.conv1d.bias,
|
||||
activation=layer.activation,
|
||||
).squeeze(-1)
|
||||
conv_state[decode_state_indices] = decode_conv_state
|
||||
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
|
||||
decode_conv_state = conv_state[decode_state_indices].contiguous()
|
||||
decode_mixed_qkv = causal_conv1d_update_torch(
|
||||
x=decode_mixed_qkv.unsqueeze(-1),
|
||||
conv_state=decode_conv_state,
|
||||
weight=conv_weights,
|
||||
bias=layer.conv1d.bias,
|
||||
activation=layer.activation,
|
||||
).squeeze(-1)
|
||||
conv_state[decode_state_indices] = decode_conv_state
|
||||
else:
|
||||
decode_mixed_qkv = causal_conv1d_update_cpu(
|
||||
x=decode_mixed_qkv,
|
||||
conv_state=conv_state,
|
||||
weight=conv_weights,
|
||||
bias=layer.conv1d.bias,
|
||||
activation=layer.activation,
|
||||
conv_state_indices=decode_state_indices,
|
||||
)
|
||||
|
||||
query, key, value = layer.rearrange_mixed_qkv(decode_mixed_qkv)
|
||||
|
||||
@@ -495,17 +508,26 @@ def _spec_aware_nonspec(
|
||||
decode_a = a[:num_decode_tokens]
|
||||
decode_state_indices = state_indices_tensor[:num_decodes]
|
||||
# Only the first ``width-1`` columns hold the real conv state.
|
||||
decode_conv_state = conv_buf[decode_state_indices][
|
||||
:, :, : width - 1
|
||||
].contiguous()
|
||||
decode_mixed_qkv = causal_conv1d_update_torch(
|
||||
x=decode_mixed_qkv.unsqueeze(-1),
|
||||
conv_state=decode_conv_state,
|
||||
weight=conv_weights,
|
||||
bias=layer.conv1d.bias,
|
||||
activation=layer.activation,
|
||||
).squeeze(-1)
|
||||
conv_buf[decode_state_indices, :, : width - 1] = decode_conv_state
|
||||
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
|
||||
conv_state_view = conv_buf[:, :, : width - 1]
|
||||
decode_conv_state = conv_state_view[decode_state_indices].contiguous()
|
||||
decode_mixed_qkv = causal_conv1d_update_torch(
|
||||
x=decode_mixed_qkv.unsqueeze(-1),
|
||||
conv_state=decode_conv_state,
|
||||
weight=conv_weights,
|
||||
bias=layer.conv1d.bias,
|
||||
activation=layer.activation,
|
||||
).squeeze(-1)
|
||||
conv_state_view[decode_state_indices] = decode_conv_state
|
||||
else:
|
||||
decode_mixed_qkv = causal_conv1d_update_cpu(
|
||||
x=decode_mixed_qkv,
|
||||
conv_state=conv_buf[:, :, : width - 1],
|
||||
weight=conv_weights,
|
||||
bias=layer.conv1d.bias,
|
||||
activation=layer.activation,
|
||||
conv_state_indices=decode_state_indices,
|
||||
)
|
||||
|
||||
query, key, value = layer.rearrange_mixed_qkv(decode_mixed_qkv)
|
||||
# rearrange_mixed_qkv can return views whose last dim is not
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
|
||||
|
||||
def _mamba_chunk_scan_combined_fwd_cpu(
|
||||
x,
|
||||
dt,
|
||||
A,
|
||||
B,
|
||||
C,
|
||||
chunk_size,
|
||||
out,
|
||||
D=None,
|
||||
z=None,
|
||||
dt_bias=None,
|
||||
initial_states=None,
|
||||
return_intermediate_states=False,
|
||||
seq_idx=None,
|
||||
cu_seqlens=None,
|
||||
cu_chunk_seqlens=None,
|
||||
last_chunk_indices=None,
|
||||
dt_softplus=False,
|
||||
dt_limit=(0.0, float("inf")),
|
||||
state_dtype=None,
|
||||
**kwargs,
|
||||
):
|
||||
seqlen, nheads, headdim = x.shape
|
||||
_, ngroups, dstate = B.shape
|
||||
|
||||
assert cu_seqlens is not None
|
||||
batch = cu_seqlens.size(0) - 1
|
||||
|
||||
dt_f = dt.float()
|
||||
if dt_bias is not None:
|
||||
dt_f = dt_f + dt_bias.float().unsqueeze(0)
|
||||
if dt_softplus:
|
||||
dt_f = torch.nn.functional.softplus(dt_f)
|
||||
if dt_limit[0] > 0.0 or dt_limit[1] < float("inf"):
|
||||
dt_f = dt_f.clamp(min=dt_limit[0], max=dt_limit[1])
|
||||
|
||||
all_states = torch.zeros(
|
||||
batch, nheads, headdim, dstate, dtype=torch.float32, device=x.device
|
||||
)
|
||||
if initial_states is not None:
|
||||
all_states.copy_(initial_states.float())
|
||||
|
||||
assert out.is_contiguous(), (
|
||||
"_mamba_chunk_scan_combined_fwd_cpu: `out` must be "
|
||||
"pre-allocated as a contiguous tensor"
|
||||
)
|
||||
|
||||
D_1d = None
|
||||
if D is not None:
|
||||
d = D.float()
|
||||
while d.dim() > 1 and d.stride(-1) == 0:
|
||||
d = d.squeeze(-1)
|
||||
D_1d = d.contiguous()
|
||||
|
||||
ops.mamba_chunk_scan_fwd_cpu(
|
||||
out,
|
||||
all_states,
|
||||
x,
|
||||
dt_f,
|
||||
A,
|
||||
B,
|
||||
C,
|
||||
D_1d,
|
||||
z,
|
||||
cu_seqlens.to(torch.int32),
|
||||
)
|
||||
|
||||
out_dtype = state_dtype if state_dtype is not None else x.dtype
|
||||
all_states = all_states.to(out_dtype)
|
||||
|
||||
return all_states
|
||||
|
||||
|
||||
def selective_state_update(
|
||||
state,
|
||||
x,
|
||||
dt,
|
||||
A,
|
||||
B,
|
||||
C,
|
||||
D=None,
|
||||
z=None,
|
||||
dt_bias=None,
|
||||
dt_softplus=False,
|
||||
state_batch_indices=None,
|
||||
dst_state_batch_indices=None,
|
||||
null_block_id=NULL_BLOCK_ID,
|
||||
out=None,
|
||||
num_accepted_tokens=None,
|
||||
cu_seqlens=None,
|
||||
is_blackwell=False,
|
||||
enable_stochastic_rounding=False,
|
||||
cache_philox_rounds=0,
|
||||
):
|
||||
"""CPU implementation for selective_state_update."""
|
||||
# Ensure out tensor exists
|
||||
if out is None:
|
||||
out = torch.empty_like(x if x.dim() == 2 else x)
|
||||
|
||||
_state = state.unsqueeze(1) if state.dim() == 3 else state
|
||||
_x = x.unsqueeze(1) if x.dim() == 2 else x
|
||||
_dt = dt.unsqueeze(1) if dt.dim() == 2 else dt
|
||||
_A = A.unsqueeze(0) if A.dim() == 2 else A
|
||||
_B = B.unsqueeze(1) if B.dim() == 2 else B
|
||||
_C = C.unsqueeze(1) if C.dim() == 2 else C
|
||||
_D = D.unsqueeze(0) if (D is not None and D.dim() == 1) else D
|
||||
_z = z.unsqueeze(1) if (z is not None and z.dim() == 2) else z
|
||||
_dt_bias = (
|
||||
dt_bias.unsqueeze(0)
|
||||
if (dt_bias is not None and dt_bias.dim() == 1)
|
||||
else dt_bias
|
||||
)
|
||||
_out = out.unsqueeze(1) if out.dim() == 2 else out
|
||||
|
||||
_sbi = state_batch_indices
|
||||
_dsbi = dst_state_batch_indices
|
||||
ops.selective_state_update_cpu(
|
||||
_state,
|
||||
_x,
|
||||
_dt,
|
||||
_A,
|
||||
_B,
|
||||
_C,
|
||||
_D,
|
||||
_z,
|
||||
_dt_bias,
|
||||
dt_softplus,
|
||||
_sbi,
|
||||
_dsbi,
|
||||
null_block_id,
|
||||
_out,
|
||||
num_accepted_tokens,
|
||||
cu_seqlens,
|
||||
)
|
||||
return _out.squeeze(1) if out.dim() == 2 else _out
|
||||
@@ -845,3 +845,13 @@ def selective_scan_fn(
|
||||
return delta # output written inplace to delta
|
||||
else:
|
||||
return z # output written inplace to z
|
||||
|
||||
|
||||
from vllm.platforms import current_platform # noqa: E402
|
||||
|
||||
if current_platform.is_cpu():
|
||||
from vllm.model_executor.layers.mamba.ops.cpu.mamba_ssm import (
|
||||
selective_state_update as selective_state_update_cpu,
|
||||
)
|
||||
|
||||
selective_state_update = selective_state_update_cpu # type: ignore
|
||||
|
||||
@@ -225,3 +225,11 @@ def mamba_chunk_scan_combined_varlen(
|
||||
)
|
||||
|
||||
return varlen_states
|
||||
|
||||
|
||||
from vllm.platforms import current_platform # noqa: E402
|
||||
|
||||
if current_platform.is_cpu():
|
||||
import vllm.model_executor.layers.mamba.ops.cpu.mamba_ssm as cpu_mamba_ssm
|
||||
|
||||
_mamba_chunk_scan_combined_fwd = cpu_mamba_ssm._mamba_chunk_scan_combined_fwd_cpu # type: ignore
|
||||
|
||||
@@ -4,8 +4,9 @@
|
||||
Dispatch module for Mamba selective state update (SSU) backends.
|
||||
|
||||
Provides a unified `selective_state_update` function that dispatches to
|
||||
either the Triton or FlashInfer backend based on the configured
|
||||
`MambaBackendEnum`. Follows SGLang's dispatch pattern adapted for vLLM.
|
||||
the Triton, FlashInfer, or CPU backend based on the configured
|
||||
`MambaBackendEnum`. On CPU-only platforms (PowerPC, x86 without CUDA)
|
||||
the backend defaults to 'cpu'.
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
@@ -182,9 +183,75 @@ class FlashInferSSUBackend(MambaSSUBackend):
|
||||
)
|
||||
|
||||
|
||||
class CPUSSUBackend(MambaSSUBackend):
|
||||
"""CPU SSU backend using the compiled C++ VSX/scalar kernel.
|
||||
|
||||
On CPU-only platforms (PowerPC, x86 without CUDA) this dispatches to
|
||||
the vectorized C++ kernel registered as ``torch.ops._C.selective_state_update_cpu``.
|
||||
That kernel uses vec_op SIMD intrinsics (VSX on ppc64le, AVX2 on x86,
|
||||
scalar fallback elsewhere) and is parallelised with OpenMP across heads.
|
||||
|
||||
Falls back to the pure-PyTorch implementation only if the C++ op is
|
||||
unavailable (e.g. a CPU-less build).
|
||||
"""
|
||||
|
||||
def __init__(self, mamba_config: MambaConfig):
|
||||
super().__init__(mamba_config)
|
||||
from vllm import _custom_ops as ops
|
||||
|
||||
self._cpp_kernel = ops.selective_state_update_cpu
|
||||
logger.info("CPUSSUBackend: using compiled C++ selective_state_update kernel.")
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "cpu"
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
state: torch.Tensor,
|
||||
x: torch.Tensor,
|
||||
dt: torch.Tensor,
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
C: torch.Tensor,
|
||||
D: torch.Tensor,
|
||||
dt_bias: torch.Tensor,
|
||||
z: torch.Tensor | None = None,
|
||||
dt_softplus: bool = False,
|
||||
state_batch_indices: torch.Tensor | None = None,
|
||||
dst_state_batch_indices: torch.Tensor | None = None,
|
||||
null_block_id: int = NULL_BLOCK_ID,
|
||||
out: torch.Tensor | None = None,
|
||||
num_accepted_tokens: torch.Tensor | None = None,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
is_blackwell: bool = False,
|
||||
) -> None:
|
||||
# C++ kernel: state shape expected as (nstates, nheads, dim, dstate)
|
||||
# The kernel writes in-place into `out` and updates `state`.
|
||||
self._cpp_kernel(
|
||||
state,
|
||||
x,
|
||||
dt,
|
||||
A,
|
||||
B,
|
||||
C,
|
||||
D,
|
||||
z,
|
||||
dt_bias,
|
||||
dt_softplus,
|
||||
state_batch_indices,
|
||||
dst_state_batch_indices,
|
||||
null_block_id,
|
||||
out,
|
||||
num_accepted_tokens,
|
||||
cu_seqlens,
|
||||
)
|
||||
|
||||
|
||||
_BACKEND_REGISTRY: dict[MambaBackendEnum, type[MambaSSUBackend]] = {
|
||||
MambaBackendEnum.TRITON: TritonSSUBackend,
|
||||
MambaBackendEnum.FLASHINFER: FlashInferSSUBackend,
|
||||
MambaBackendEnum.CPU: CPUSSUBackend,
|
||||
}
|
||||
|
||||
_mamba_ssu_backend: MambaSSUBackend | None = None
|
||||
@@ -210,6 +277,20 @@ def initialize_mamba_ssu_backend(
|
||||
global _mamba_ssu_backend
|
||||
|
||||
backend = mamba_config.backend
|
||||
|
||||
# On CPU-only platforms (PowerPC, x86 without CUDA) Triton JIT is
|
||||
# unstable or unavailable. Silently fall back to the CPU
|
||||
# backend unless the user explicitly chose something other than "triton".
|
||||
if backend == MambaBackendEnum.TRITON:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_cpu():
|
||||
logger.info(
|
||||
"CPU platform detected: overriding Mamba SSU backend "
|
||||
"from 'triton' to 'cpu'."
|
||||
)
|
||||
backend = MambaBackendEnum.CPU
|
||||
|
||||
if backend not in _BACKEND_REGISTRY:
|
||||
raise ValueError(
|
||||
f"Unknown Mamba SSU backend: {backend}. "
|
||||
|
||||
@@ -94,9 +94,13 @@ class ShortConv(MambaBase, CustomOp):
|
||||
# Reference torch causal conv1d; runs on all CPU platforms. AMX kernels
|
||||
# for causal conv can be plugged in here later.
|
||||
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
|
||||
causal_conv1d_torch,
|
||||
causal_conv1d_fn_cpu as causal_conv1d_torch,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
|
||||
causal_conv1d_update_cpu,
|
||||
causal_conv1d_update_torch,
|
||||
)
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
@@ -164,17 +168,26 @@ class ShortConv(MambaBase, CustomOp):
|
||||
if has_decode:
|
||||
assert attn_metadata.state_indices_tensor_d is not None
|
||||
state_indices_d = attn_metadata.state_indices_tensor_d.flatten()
|
||||
Bx_d = (B_d * x_d).unsqueeze(-1) # (num_decodes, dim, 1)
|
||||
# Advanced indexing returns a copy; update in-place then scatter back
|
||||
gathered = conv_state[state_indices_d] # (num_decodes, dim, state_len)
|
||||
out_d = causal_conv1d_update_torch(
|
||||
Bx_d,
|
||||
gathered,
|
||||
conv_weights,
|
||||
self.conv.bias,
|
||||
activation=None,
|
||||
).squeeze(-1) # (num_decodes, dim)
|
||||
conv_state[state_indices_d] = gathered
|
||||
Bx_d = B_d * x_d # (num_decodes, dim)
|
||||
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
|
||||
conv_state_view = conv_state[state_indices_d].contiguous()
|
||||
out_d = causal_conv1d_update_torch(
|
||||
Bx_d.unsqueeze(-1),
|
||||
conv_state_view,
|
||||
conv_weights,
|
||||
self.conv.bias,
|
||||
activation=None,
|
||||
).squeeze(-1)
|
||||
conv_state[state_indices_d] = conv_state_view
|
||||
else:
|
||||
out_d = causal_conv1d_update_cpu(
|
||||
Bx_d,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
self.conv.bias,
|
||||
activation=None,
|
||||
conv_state_indices=state_indices_d,
|
||||
)
|
||||
conv_output_list.insert(0, C_d * out_d)
|
||||
|
||||
hidden_states_out = torch.vstack(conv_output_list)
|
||||
|
||||
@@ -234,10 +234,14 @@ def dispatch_cpu_unquantized_gemm(
|
||||
layer.cpu_linear = torch.nn.functional.linear
|
||||
return
|
||||
|
||||
# Skip CPU GEMM dispatch for non-2D weights (e.g. MoE 3D expert weights).
|
||||
# These layers are handled by their own specialized methods.
|
||||
if layer.weight.ndim != 2:
|
||||
# this is not a linear layer
|
||||
# For now it should be a causal_conv1d op
|
||||
if torch.cpu._is_amx_tile_supported():
|
||||
# For now it should be a causal_conv1d op or MoE 3D expert weights
|
||||
if torch.cpu._is_amx_tile_supported() and hasattr(
|
||||
ops, "causal_conv1d_weight_pack"
|
||||
):
|
||||
# prepack conv weight
|
||||
unpacked = (
|
||||
layer.weight.view(
|
||||
|
||||
Reference in New Issue
Block a user