feat(cpu): add CPU support for Mamba ShortConv (#35059)
Signed-off-by: Rahul Vishwakarma <Rahul.Vishwakarma2@ibm.com>
This commit is contained in:
@@ -17,12 +17,14 @@ steps:
|
||||
- tests/kernels/test_awq_int4_to_int8.py
|
||||
- tests/kernels/quantization/test_cpu_fp8_scaled_mm.py
|
||||
- tests/kernels/mamba/cpu/test_cpu_gdn_ops.py
|
||||
- tests/kernels/mamba/test_cpu_short_conv.py
|
||||
commands:
|
||||
- |
|
||||
bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m "
|
||||
pytest -x -v -s tests/kernels/attention/test_cpu_attn.py
|
||||
pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py
|
||||
pytest -x -v -s tests/kernels/moe/test_cpu_quant_fused_moe.py
|
||||
pytest -x -v -s tests/kernels/mamba/test_cpu_short_conv.py
|
||||
pytest -x -v -s tests/kernels/test_onednn.py
|
||||
pytest -x -v -s tests/kernels/test_awq_int4_to_int8.py
|
||||
pytest -x -v -s tests/kernels/quantization/test_cpu_fp8_scaled_mm.py
|
||||
|
||||
@@ -39,7 +39,8 @@ function cpu_tests() {
|
||||
pytest -x -v -s tests/kernels/core/test_cpu_activation.py
|
||||
pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py
|
||||
pytest -x -v -s tests/kernels/mamba/cpu/test_cpu_gdn_ops.py
|
||||
pytest -x -v -s tests/kernels/moe/test_cpu_int4_moe.py"
|
||||
pytest -x -v -s tests/kernels/moe/test_cpu_int4_moe.py
|
||||
pytest -x -v -s tests/kernels/mamba/test_cpu_short_conv.py"
|
||||
|
||||
# skip tests requiring model downloads if HF_TOKEN is not set
|
||||
# due to rate-limits
|
||||
@@ -63,7 +64,6 @@ function cpu_tests() {
|
||||
set -e
|
||||
pytest -x -v -s tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_logprobs"
|
||||
|
||||
|
||||
# basic online serving
|
||||
docker exec cpu-test bash -c '
|
||||
set -e
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import CompilationConfig, VllmConfig
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.mamba.short_conv import ShortConv
|
||||
from vllm.model_executor.layers.utils import dispatch_cpu_unquantized_gemm
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.short_conv_attn import ShortConvAttentionMetadata
|
||||
|
||||
if not current_platform.is_cpu():
|
||||
pytest.skip("skipping CPU-only tests", allow_module_level=True)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def mock_dist():
|
||||
with (
|
||||
patch(
|
||||
"vllm.model_executor.layers.linear.get_tensor_model_parallel_rank",
|
||||
return_value=0,
|
||||
),
|
||||
patch(
|
||||
"vllm.model_executor.layers.linear.get_tensor_model_parallel_world_size",
|
||||
return_value=1,
|
||||
),
|
||||
patch(
|
||||
"vllm.distributed.parallel_state.model_parallel_is_initialized",
|
||||
return_value=True,
|
||||
),
|
||||
patch(
|
||||
"vllm.distributed.parallel_state.get_tp_group",
|
||||
return_value=MagicMock(rank_in_group=0),
|
||||
),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def vllm_config():
|
||||
# ShortConv only needs compilation_config from the current vLLM config, so a
|
||||
# minimal config (model_config=None) avoids mocking ModelConfig and the
|
||||
# associated VllmConfig validation churn.
|
||||
return VllmConfig(compilation_config=CompilationConfig())
|
||||
|
||||
|
||||
def test_short_conv_forward_native_prefill(vllm_config):
|
||||
prefix = "test_layer"
|
||||
config = SimpleNamespace(conv_L_cache=4, conv_bias=True)
|
||||
dim = 16
|
||||
|
||||
from vllm.config import set_current_vllm_config
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
layer = ShortConv(config=config, dim=dim, layer_idx=0, prefix=prefix)
|
||||
|
||||
layer.to("cpu")
|
||||
dispatch_cpu_unquantized_gemm(layer.in_proj, remove_weight=False)
|
||||
dispatch_cpu_unquantized_gemm(layer.out_proj, remove_weight=False)
|
||||
|
||||
# Mock AttentionMetadata
|
||||
num_prefills = 1
|
||||
num_prefill_tokens = 5
|
||||
query_start_loc_p = torch.tensor([0, 5], dtype=torch.int32)
|
||||
state_indices_tensor_p = torch.tensor([0], dtype=torch.int32)
|
||||
|
||||
# ShortConvAttentionMetadata
|
||||
attn_metadata = ShortConvAttentionMetadata(
|
||||
num_prefills=num_prefills,
|
||||
num_prefill_tokens=num_prefill_tokens,
|
||||
num_decodes=0,
|
||||
num_decode_tokens=0,
|
||||
num_reqs=1,
|
||||
query_start_loc_p=query_start_loc_p,
|
||||
has_initial_states_p=torch.tensor([False]),
|
||||
state_indices_tensor_p=state_indices_tensor_p,
|
||||
state_indices_tensor_d=torch.empty((0, 1), dtype=torch.int32),
|
||||
num_accepted_tokens=None,
|
||||
query_start_loc_d=None,
|
||||
block_idx_last_scheduled_token=None,
|
||||
block_idx_first_scheduled_token_p=None,
|
||||
block_idx_last_computed_token=None,
|
||||
block_idx_last_scheduled_token_prev_step=None,
|
||||
num_computed_tokens_p=None,
|
||||
seq_lens=torch.tensor([5]),
|
||||
)
|
||||
|
||||
# Mock KV cache
|
||||
# conv_state shape (num_blocks, L_cache - 1, dim)
|
||||
conv_state = torch.zeros((1, config.conv_L_cache - 1, dim))
|
||||
layer.kv_cache = (conv_state,)
|
||||
|
||||
hidden_states = torch.randn((num_prefill_tokens, dim))
|
||||
output = torch.zeros_like(hidden_states)
|
||||
|
||||
attn_metadata_dict = {prefix: attn_metadata}
|
||||
with set_forward_context(attn_metadata=attn_metadata_dict, vllm_config=vllm_config):
|
||||
layer.forward_native(hidden_states, output)
|
||||
|
||||
# Check if KV cache was updated
|
||||
assert not torch.allclose(conv_state, torch.zeros_like(conv_state))
|
||||
|
||||
|
||||
def test_short_conv_forward_native_decode(vllm_config):
|
||||
prefix = "test_layer_decode"
|
||||
config = SimpleNamespace(conv_L_cache=4, conv_bias=True)
|
||||
dim = 16
|
||||
|
||||
from vllm.config import set_current_vllm_config
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
layer = ShortConv(config=config, dim=dim, layer_idx=0, prefix=prefix)
|
||||
|
||||
layer.to("cpu")
|
||||
dispatch_cpu_unquantized_gemm(layer.in_proj, remove_weight=False)
|
||||
dispatch_cpu_unquantized_gemm(layer.out_proj, remove_weight=False)
|
||||
|
||||
# Mock AttentionMetadata for 2 decode requests
|
||||
num_decodes = 2
|
||||
state_indices_tensor_d = torch.tensor([0, 1], dtype=torch.int32)
|
||||
|
||||
attn_metadata = ShortConvAttentionMetadata(
|
||||
num_prefills=0,
|
||||
num_prefill_tokens=0,
|
||||
num_decodes=num_decodes,
|
||||
num_decode_tokens=num_decodes,
|
||||
num_reqs=num_decodes,
|
||||
query_start_loc_p=None,
|
||||
has_initial_states_p=None,
|
||||
state_indices_tensor_p=torch.empty((0,), dtype=torch.int32),
|
||||
state_indices_tensor_d=state_indices_tensor_d,
|
||||
num_accepted_tokens=None,
|
||||
query_start_loc_d=torch.tensor([0, 1, 2], dtype=torch.int32),
|
||||
block_idx_last_scheduled_token=None,
|
||||
block_idx_first_scheduled_token_p=None,
|
||||
block_idx_last_computed_token=None,
|
||||
block_idx_last_scheduled_token_prev_step=None,
|
||||
num_computed_tokens_p=None,
|
||||
seq_lens=torch.tensor([1, 1]),
|
||||
)
|
||||
|
||||
# Mock KV cache (2 blocks for 2 requests)
|
||||
conv_state = torch.randn((2, config.conv_L_cache - 1, dim))
|
||||
layer.kv_cache = (conv_state,)
|
||||
|
||||
hidden_states = torch.randn((num_decodes, dim))
|
||||
output = torch.zeros_like(hidden_states)
|
||||
|
||||
old_conv_state = conv_state.clone()
|
||||
|
||||
attn_metadata_dict = {prefix: attn_metadata}
|
||||
with set_forward_context(attn_metadata=attn_metadata_dict, vllm_config=vllm_config):
|
||||
layer.forward_native(hidden_states, output)
|
||||
|
||||
# Check if KV cache was updated
|
||||
assert not torch.allclose(conv_state, old_conv_state)
|
||||
|
||||
|
||||
def test_dispatch_cpu_unquantized_gemm_conv_layer():
|
||||
# Convolution layers have >2D weights; dispatch should skip them gracefully.
|
||||
# Shape/dtype are AMX-pack safe (bf16, width==4, dim % block_size == 0) so
|
||||
# the AMX prepack branch does not raise on AMX-capable CPUs.
|
||||
class MockConvLayer(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.weight = torch.nn.Parameter(
|
||||
torch.randn(32, 1, 4, dtype=torch.bfloat16)
|
||||
)
|
||||
self.bias = torch.nn.Parameter(torch.randn(32, dtype=torch.bfloat16))
|
||||
|
||||
layer = MockConvLayer()
|
||||
# The ndim != 2 guard returns early without raising.
|
||||
dispatch_cpu_unquantized_gemm(layer, remove_weight=False)
|
||||
# No cpu_linear set — conv layers are handled elsewhere.
|
||||
assert not hasattr(layer, "cpu_linear")
|
||||
@@ -23,6 +23,7 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
causal_conv1d_fn,
|
||||
causal_conv1d_update,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
@@ -90,7 +91,94 @@ class ShortConv(MambaBase, CustomOp):
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
):
|
||||
return
|
||||
# Reference torch causal conv1d; runs on all CPU platforms. AMX kernels
|
||||
# for causal conv can be plugged in here later.
|
||||
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
|
||||
causal_conv1d_torch,
|
||||
causal_conv1d_update_torch,
|
||||
)
|
||||
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
attn_metadata: AttentionMetadata | None = None
|
||||
if attn_metadata_raw is not None:
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata = attn_metadata_raw[self.prefix]
|
||||
assert isinstance(attn_metadata, ShortConvAttentionMetadata)
|
||||
|
||||
BCx, _ = self.in_proj(hidden_states)
|
||||
B, C, x = BCx.chunk(3, dim=-1)
|
||||
|
||||
# (dim, kernel_size) — same reshape as forward_cuda
|
||||
conv_weights = self.conv.weight.view(
|
||||
self.conv.weight.size(0), self.conv.weight.size(2)
|
||||
)
|
||||
|
||||
if attn_metadata is None:
|
||||
# Profile run — output value doesn't matter
|
||||
Bx = (B * x).contiguous()
|
||||
output_tensor, _ = self.out_proj(C * Bx)
|
||||
output[: hidden_states.shape[0]] = output_tensor
|
||||
return
|
||||
|
||||
conv_state = (
|
||||
self.kv_cache[0]
|
||||
if is_conv_state_dim_first()
|
||||
else self.kv_cache[0].transpose(-1, -2)
|
||||
) # (num_blocks, dim, state_len)
|
||||
|
||||
num_prefills = attn_metadata.num_prefills
|
||||
num_decodes = attn_metadata.num_decode_tokens
|
||||
num_prefill_tokens = attn_metadata.num_prefill_tokens
|
||||
has_prefill = num_prefills > 0
|
||||
has_decode = num_decodes > 0
|
||||
num_actual_tokens = num_decodes + num_prefill_tokens
|
||||
|
||||
B_d, B_p = torch.split(
|
||||
B[:num_actual_tokens], [num_decodes, num_prefill_tokens], dim=0
|
||||
)
|
||||
C_d, C_p = torch.split(
|
||||
C[:num_actual_tokens], [num_decodes, num_prefill_tokens], dim=0
|
||||
)
|
||||
x_d, x_p = torch.split(
|
||||
x[:num_actual_tokens], [num_decodes, num_prefill_tokens], dim=0
|
||||
)
|
||||
|
||||
conv_output_list = []
|
||||
|
||||
if has_prefill:
|
||||
assert attn_metadata.state_indices_tensor_p is not None
|
||||
Bx_p = (B_p * x_p).transpose(0, 1) # (dim, num_prefill_tokens)
|
||||
out_p = causal_conv1d_torch(
|
||||
Bx_p,
|
||||
conv_weights,
|
||||
self.conv.bias,
|
||||
conv_state,
|
||||
attn_metadata.query_start_loc_p,
|
||||
attn_metadata.state_indices_tensor_p.flatten(),
|
||||
attn_metadata.has_initial_states_p,
|
||||
activation=None,
|
||||
).transpose(0, 1)[:num_prefill_tokens] # (num_prefill_tokens, dim)
|
||||
conv_output_list.append(C_p * out_p)
|
||||
|
||||
if has_decode:
|
||||
assert attn_metadata.state_indices_tensor_d is not None
|
||||
state_indices_d = attn_metadata.state_indices_tensor_d.flatten()
|
||||
Bx_d = (B_d * x_d).unsqueeze(-1) # (num_decodes, dim, 1)
|
||||
# Advanced indexing returns a copy; update in-place then scatter back
|
||||
gathered = conv_state[state_indices_d] # (num_decodes, dim, state_len)
|
||||
out_d = causal_conv1d_update_torch(
|
||||
Bx_d,
|
||||
gathered,
|
||||
conv_weights,
|
||||
self.conv.bias,
|
||||
activation=None,
|
||||
).squeeze(-1) # (num_decodes, dim)
|
||||
conv_state[state_indices_d] = gathered
|
||||
conv_output_list.insert(0, C_d * out_d)
|
||||
|
||||
hidden_states_out = torch.vstack(conv_output_list)
|
||||
output[:num_actual_tokens], _ = self.out_proj(hidden_states_out)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -235,7 +323,10 @@ def short_conv(
|
||||
) -> None:
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
self = forward_context.no_compile_layers[layer_name]
|
||||
self.forward_cuda(hidden_states=hidden_states, output=output)
|
||||
if not current_platform.is_cpu():
|
||||
self.forward_cuda(hidden_states=hidden_states, output=output)
|
||||
else:
|
||||
self.forward_native(hidden_states=hidden_states, output=output)
|
||||
|
||||
|
||||
def short_conv_fake(
|
||||
|
||||
Reference in New Issue
Block a user