feat(cpu): add CPU support for Mamba ShortConv (#35059)

Signed-off-by: Rahul Vishwakarma <Rahul.Vishwakarma2@ibm.com>
This commit is contained in:
Rahul Vishwakarma
2026-07-07 10:47:08 +08:00
committed by GitHub
parent a46c9329e5
commit 9dd2465896
4 changed files with 277 additions and 4 deletions
+2
View File
@@ -17,12 +17,14 @@ steps:
- tests/kernels/test_awq_int4_to_int8.py
- tests/kernels/quantization/test_cpu_fp8_scaled_mm.py
- tests/kernels/mamba/cpu/test_cpu_gdn_ops.py
- tests/kernels/mamba/test_cpu_short_conv.py
commands:
- |
bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m "
pytest -x -v -s tests/kernels/attention/test_cpu_attn.py
pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py
pytest -x -v -s tests/kernels/moe/test_cpu_quant_fused_moe.py
pytest -x -v -s tests/kernels/mamba/test_cpu_short_conv.py
pytest -x -v -s tests/kernels/test_onednn.py
pytest -x -v -s tests/kernels/test_awq_int4_to_int8.py
pytest -x -v -s tests/kernels/quantization/test_cpu_fp8_scaled_mm.py
@@ -39,7 +39,8 @@ function cpu_tests() {
pytest -x -v -s tests/kernels/core/test_cpu_activation.py
pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py
pytest -x -v -s tests/kernels/mamba/cpu/test_cpu_gdn_ops.py
pytest -x -v -s tests/kernels/moe/test_cpu_int4_moe.py"
pytest -x -v -s tests/kernels/moe/test_cpu_int4_moe.py
pytest -x -v -s tests/kernels/mamba/test_cpu_short_conv.py"
# skip tests requiring model downloads if HF_TOKEN is not set
# due to rate-limits
@@ -63,7 +64,6 @@ function cpu_tests() {
set -e
pytest -x -v -s tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_logprobs"
# basic online serving
docker exec cpu-test bash -c '
set -e
+180
View File
@@ -0,0 +1,180 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
import torch
from vllm.config import CompilationConfig, VllmConfig
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.mamba.short_conv import ShortConv
from vllm.model_executor.layers.utils import dispatch_cpu_unquantized_gemm
from vllm.platforms import current_platform
from vllm.v1.attention.backends.short_conv_attn import ShortConvAttentionMetadata
if not current_platform.is_cpu():
pytest.skip("skipping CPU-only tests", allow_module_level=True)
@pytest.fixture(autouse=True)
def mock_dist():
with (
patch(
"vllm.model_executor.layers.linear.get_tensor_model_parallel_rank",
return_value=0,
),
patch(
"vllm.model_executor.layers.linear.get_tensor_model_parallel_world_size",
return_value=1,
),
patch(
"vllm.distributed.parallel_state.model_parallel_is_initialized",
return_value=True,
),
patch(
"vllm.distributed.parallel_state.get_tp_group",
return_value=MagicMock(rank_in_group=0),
),
):
yield
@pytest.fixture
def vllm_config():
# ShortConv only needs compilation_config from the current vLLM config, so a
# minimal config (model_config=None) avoids mocking ModelConfig and the
# associated VllmConfig validation churn.
return VllmConfig(compilation_config=CompilationConfig())
def test_short_conv_forward_native_prefill(vllm_config):
prefix = "test_layer"
config = SimpleNamespace(conv_L_cache=4, conv_bias=True)
dim = 16
from vllm.config import set_current_vllm_config
with set_current_vllm_config(vllm_config):
layer = ShortConv(config=config, dim=dim, layer_idx=0, prefix=prefix)
layer.to("cpu")
dispatch_cpu_unquantized_gemm(layer.in_proj, remove_weight=False)
dispatch_cpu_unquantized_gemm(layer.out_proj, remove_weight=False)
# Mock AttentionMetadata
num_prefills = 1
num_prefill_tokens = 5
query_start_loc_p = torch.tensor([0, 5], dtype=torch.int32)
state_indices_tensor_p = torch.tensor([0], dtype=torch.int32)
# ShortConvAttentionMetadata
attn_metadata = ShortConvAttentionMetadata(
num_prefills=num_prefills,
num_prefill_tokens=num_prefill_tokens,
num_decodes=0,
num_decode_tokens=0,
num_reqs=1,
query_start_loc_p=query_start_loc_p,
has_initial_states_p=torch.tensor([False]),
state_indices_tensor_p=state_indices_tensor_p,
state_indices_tensor_d=torch.empty((0, 1), dtype=torch.int32),
num_accepted_tokens=None,
query_start_loc_d=None,
block_idx_last_scheduled_token=None,
block_idx_first_scheduled_token_p=None,
block_idx_last_computed_token=None,
block_idx_last_scheduled_token_prev_step=None,
num_computed_tokens_p=None,
seq_lens=torch.tensor([5]),
)
# Mock KV cache
# conv_state shape (num_blocks, L_cache - 1, dim)
conv_state = torch.zeros((1, config.conv_L_cache - 1, dim))
layer.kv_cache = (conv_state,)
hidden_states = torch.randn((num_prefill_tokens, dim))
output = torch.zeros_like(hidden_states)
attn_metadata_dict = {prefix: attn_metadata}
with set_forward_context(attn_metadata=attn_metadata_dict, vllm_config=vllm_config):
layer.forward_native(hidden_states, output)
# Check if KV cache was updated
assert not torch.allclose(conv_state, torch.zeros_like(conv_state))
def test_short_conv_forward_native_decode(vllm_config):
prefix = "test_layer_decode"
config = SimpleNamespace(conv_L_cache=4, conv_bias=True)
dim = 16
from vllm.config import set_current_vllm_config
with set_current_vllm_config(vllm_config):
layer = ShortConv(config=config, dim=dim, layer_idx=0, prefix=prefix)
layer.to("cpu")
dispatch_cpu_unquantized_gemm(layer.in_proj, remove_weight=False)
dispatch_cpu_unquantized_gemm(layer.out_proj, remove_weight=False)
# Mock AttentionMetadata for 2 decode requests
num_decodes = 2
state_indices_tensor_d = torch.tensor([0, 1], dtype=torch.int32)
attn_metadata = ShortConvAttentionMetadata(
num_prefills=0,
num_prefill_tokens=0,
num_decodes=num_decodes,
num_decode_tokens=num_decodes,
num_reqs=num_decodes,
query_start_loc_p=None,
has_initial_states_p=None,
state_indices_tensor_p=torch.empty((0,), dtype=torch.int32),
state_indices_tensor_d=state_indices_tensor_d,
num_accepted_tokens=None,
query_start_loc_d=torch.tensor([0, 1, 2], dtype=torch.int32),
block_idx_last_scheduled_token=None,
block_idx_first_scheduled_token_p=None,
block_idx_last_computed_token=None,
block_idx_last_scheduled_token_prev_step=None,
num_computed_tokens_p=None,
seq_lens=torch.tensor([1, 1]),
)
# Mock KV cache (2 blocks for 2 requests)
conv_state = torch.randn((2, config.conv_L_cache - 1, dim))
layer.kv_cache = (conv_state,)
hidden_states = torch.randn((num_decodes, dim))
output = torch.zeros_like(hidden_states)
old_conv_state = conv_state.clone()
attn_metadata_dict = {prefix: attn_metadata}
with set_forward_context(attn_metadata=attn_metadata_dict, vllm_config=vllm_config):
layer.forward_native(hidden_states, output)
# Check if KV cache was updated
assert not torch.allclose(conv_state, old_conv_state)
def test_dispatch_cpu_unquantized_gemm_conv_layer():
# Convolution layers have >2D weights; dispatch should skip them gracefully.
# Shape/dtype are AMX-pack safe (bf16, width==4, dim % block_size == 0) so
# the AMX prepack branch does not raise on AMX-capable CPUs.
class MockConvLayer(torch.nn.Module):
def __init__(self):
super().__init__()
self.weight = torch.nn.Parameter(
torch.randn(32, 1, 4, dtype=torch.bfloat16)
)
self.bias = torch.nn.Parameter(torch.randn(32, dtype=torch.bfloat16))
layer = MockConvLayer()
# The ndim != 2 guard returns early without raising.
dispatch_cpu_unquantized_gemm(layer, remove_weight=False)
# No cpu_linear set — conv layers are handled elsewhere.
assert not hasattr(layer, "cpu_linear")
+93 -2
View File
@@ -23,6 +23,7 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
causal_conv1d_fn,
causal_conv1d_update,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
@@ -90,7 +91,94 @@ class ShortConv(MambaBase, CustomOp):
hidden_states: torch.Tensor,
output: torch.Tensor,
):
return
# Reference torch causal conv1d; runs on all CPU platforms. AMX kernels
# for causal conv can be plugged in here later.
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
causal_conv1d_torch,
causal_conv1d_update_torch,
)
forward_context = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata
attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None:
assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw[self.prefix]
assert isinstance(attn_metadata, ShortConvAttentionMetadata)
BCx, _ = self.in_proj(hidden_states)
B, C, x = BCx.chunk(3, dim=-1)
# (dim, kernel_size) — same reshape as forward_cuda
conv_weights = self.conv.weight.view(
self.conv.weight.size(0), self.conv.weight.size(2)
)
if attn_metadata is None:
# Profile run — output value doesn't matter
Bx = (B * x).contiguous()
output_tensor, _ = self.out_proj(C * Bx)
output[: hidden_states.shape[0]] = output_tensor
return
conv_state = (
self.kv_cache[0]
if is_conv_state_dim_first()
else self.kv_cache[0].transpose(-1, -2)
) # (num_blocks, dim, state_len)
num_prefills = attn_metadata.num_prefills
num_decodes = attn_metadata.num_decode_tokens
num_prefill_tokens = attn_metadata.num_prefill_tokens
has_prefill = num_prefills > 0
has_decode = num_decodes > 0
num_actual_tokens = num_decodes + num_prefill_tokens
B_d, B_p = torch.split(
B[:num_actual_tokens], [num_decodes, num_prefill_tokens], dim=0
)
C_d, C_p = torch.split(
C[:num_actual_tokens], [num_decodes, num_prefill_tokens], dim=0
)
x_d, x_p = torch.split(
x[:num_actual_tokens], [num_decodes, num_prefill_tokens], dim=0
)
conv_output_list = []
if has_prefill:
assert attn_metadata.state_indices_tensor_p is not None
Bx_p = (B_p * x_p).transpose(0, 1) # (dim, num_prefill_tokens)
out_p = causal_conv1d_torch(
Bx_p,
conv_weights,
self.conv.bias,
conv_state,
attn_metadata.query_start_loc_p,
attn_metadata.state_indices_tensor_p.flatten(),
attn_metadata.has_initial_states_p,
activation=None,
).transpose(0, 1)[:num_prefill_tokens] # (num_prefill_tokens, dim)
conv_output_list.append(C_p * out_p)
if has_decode:
assert attn_metadata.state_indices_tensor_d is not None
state_indices_d = attn_metadata.state_indices_tensor_d.flatten()
Bx_d = (B_d * x_d).unsqueeze(-1) # (num_decodes, dim, 1)
# Advanced indexing returns a copy; update in-place then scatter back
gathered = conv_state[state_indices_d] # (num_decodes, dim, state_len)
out_d = causal_conv1d_update_torch(
Bx_d,
gathered,
conv_weights,
self.conv.bias,
activation=None,
).squeeze(-1) # (num_decodes, dim)
conv_state[state_indices_d] = gathered
conv_output_list.insert(0, C_d * out_d)
hidden_states_out = torch.vstack(conv_output_list)
output[:num_actual_tokens], _ = self.out_proj(hidden_states_out)
def forward(
self,
@@ -235,7 +323,10 @@ def short_conv(
) -> None:
forward_context: ForwardContext = get_forward_context()
self = forward_context.no_compile_layers[layer_name]
self.forward_cuda(hidden_states=hidden_states, output=output)
if not current_platform.is_cpu():
self.forward_cuda(hidden_states=hidden_states, output=output)
else:
self.forward_native(hidden_states=hidden_states, output=output)
def short_conv_fake(