diff --git a/.buildkite/hardware_tests/cpu.yaml b/.buildkite/hardware_tests/cpu.yaml index 6f1bd34454..b34585f253 100644 --- a/.buildkite/hardware_tests/cpu.yaml +++ b/.buildkite/hardware_tests/cpu.yaml @@ -17,12 +17,14 @@ steps: - tests/kernels/test_awq_int4_to_int8.py - tests/kernels/quantization/test_cpu_fp8_scaled_mm.py - tests/kernels/mamba/cpu/test_cpu_gdn_ops.py + - tests/kernels/mamba/test_cpu_short_conv.py commands: - | bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m " pytest -x -v -s tests/kernels/attention/test_cpu_attn.py pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py pytest -x -v -s tests/kernels/moe/test_cpu_quant_fused_moe.py + pytest -x -v -s tests/kernels/mamba/test_cpu_short_conv.py pytest -x -v -s tests/kernels/test_onednn.py pytest -x -v -s tests/kernels/test_awq_int4_to_int8.py pytest -x -v -s tests/kernels/quantization/test_cpu_fp8_scaled_mm.py diff --git a/.buildkite/scripts/hardware_ci/run-cpu-test-arm.sh b/.buildkite/scripts/hardware_ci/run-cpu-test-arm.sh index 2ad3a16be8..2d11dd477e 100755 --- a/.buildkite/scripts/hardware_ci/run-cpu-test-arm.sh +++ b/.buildkite/scripts/hardware_ci/run-cpu-test-arm.sh @@ -39,7 +39,8 @@ function cpu_tests() { pytest -x -v -s tests/kernels/core/test_cpu_activation.py pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py pytest -x -v -s tests/kernels/mamba/cpu/test_cpu_gdn_ops.py - pytest -x -v -s tests/kernels/moe/test_cpu_int4_moe.py" + pytest -x -v -s tests/kernels/moe/test_cpu_int4_moe.py + pytest -x -v -s tests/kernels/mamba/test_cpu_short_conv.py" # skip tests requiring model downloads if HF_TOKEN is not set # due to rate-limits @@ -63,7 +64,6 @@ function cpu_tests() { set -e pytest -x -v -s tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_logprobs" - # basic online serving docker exec cpu-test bash -c ' set -e diff --git a/tests/kernels/mamba/test_cpu_short_conv.py b/tests/kernels/mamba/test_cpu_short_conv.py new file mode 100644 index 0000000000..cd32e0901a --- /dev/null +++ b/tests/kernels/mamba/test_cpu_short_conv.py @@ -0,0 +1,180 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest +import torch + +from vllm.config import CompilationConfig, VllmConfig +from vllm.forward_context import set_forward_context +from vllm.model_executor.layers.mamba.short_conv import ShortConv +from vllm.model_executor.layers.utils import dispatch_cpu_unquantized_gemm +from vllm.platforms import current_platform +from vllm.v1.attention.backends.short_conv_attn import ShortConvAttentionMetadata + +if not current_platform.is_cpu(): + pytest.skip("skipping CPU-only tests", allow_module_level=True) + + +@pytest.fixture(autouse=True) +def mock_dist(): + with ( + patch( + "vllm.model_executor.layers.linear.get_tensor_model_parallel_rank", + return_value=0, + ), + patch( + "vllm.model_executor.layers.linear.get_tensor_model_parallel_world_size", + return_value=1, + ), + patch( + "vllm.distributed.parallel_state.model_parallel_is_initialized", + return_value=True, + ), + patch( + "vllm.distributed.parallel_state.get_tp_group", + return_value=MagicMock(rank_in_group=0), + ), + ): + yield + + +@pytest.fixture +def vllm_config(): + # ShortConv only needs compilation_config from the current vLLM config, so a + # minimal config (model_config=None) avoids mocking ModelConfig and the + # associated VllmConfig validation churn. + return VllmConfig(compilation_config=CompilationConfig()) + + +def test_short_conv_forward_native_prefill(vllm_config): + prefix = "test_layer" + config = SimpleNamespace(conv_L_cache=4, conv_bias=True) + dim = 16 + + from vllm.config import set_current_vllm_config + + with set_current_vllm_config(vllm_config): + layer = ShortConv(config=config, dim=dim, layer_idx=0, prefix=prefix) + + layer.to("cpu") + dispatch_cpu_unquantized_gemm(layer.in_proj, remove_weight=False) + dispatch_cpu_unquantized_gemm(layer.out_proj, remove_weight=False) + + # Mock AttentionMetadata + num_prefills = 1 + num_prefill_tokens = 5 + query_start_loc_p = torch.tensor([0, 5], dtype=torch.int32) + state_indices_tensor_p = torch.tensor([0], dtype=torch.int32) + + # ShortConvAttentionMetadata + attn_metadata = ShortConvAttentionMetadata( + num_prefills=num_prefills, + num_prefill_tokens=num_prefill_tokens, + num_decodes=0, + num_decode_tokens=0, + num_reqs=1, + query_start_loc_p=query_start_loc_p, + has_initial_states_p=torch.tensor([False]), + state_indices_tensor_p=state_indices_tensor_p, + state_indices_tensor_d=torch.empty((0, 1), dtype=torch.int32), + num_accepted_tokens=None, + query_start_loc_d=None, + block_idx_last_scheduled_token=None, + block_idx_first_scheduled_token_p=None, + block_idx_last_computed_token=None, + block_idx_last_scheduled_token_prev_step=None, + num_computed_tokens_p=None, + seq_lens=torch.tensor([5]), + ) + + # Mock KV cache + # conv_state shape (num_blocks, L_cache - 1, dim) + conv_state = torch.zeros((1, config.conv_L_cache - 1, dim)) + layer.kv_cache = (conv_state,) + + hidden_states = torch.randn((num_prefill_tokens, dim)) + output = torch.zeros_like(hidden_states) + + attn_metadata_dict = {prefix: attn_metadata} + with set_forward_context(attn_metadata=attn_metadata_dict, vllm_config=vllm_config): + layer.forward_native(hidden_states, output) + + # Check if KV cache was updated + assert not torch.allclose(conv_state, torch.zeros_like(conv_state)) + + +def test_short_conv_forward_native_decode(vllm_config): + prefix = "test_layer_decode" + config = SimpleNamespace(conv_L_cache=4, conv_bias=True) + dim = 16 + + from vllm.config import set_current_vllm_config + + with set_current_vllm_config(vllm_config): + layer = ShortConv(config=config, dim=dim, layer_idx=0, prefix=prefix) + + layer.to("cpu") + dispatch_cpu_unquantized_gemm(layer.in_proj, remove_weight=False) + dispatch_cpu_unquantized_gemm(layer.out_proj, remove_weight=False) + + # Mock AttentionMetadata for 2 decode requests + num_decodes = 2 + state_indices_tensor_d = torch.tensor([0, 1], dtype=torch.int32) + + attn_metadata = ShortConvAttentionMetadata( + num_prefills=0, + num_prefill_tokens=0, + num_decodes=num_decodes, + num_decode_tokens=num_decodes, + num_reqs=num_decodes, + query_start_loc_p=None, + has_initial_states_p=None, + state_indices_tensor_p=torch.empty((0,), dtype=torch.int32), + state_indices_tensor_d=state_indices_tensor_d, + num_accepted_tokens=None, + query_start_loc_d=torch.tensor([0, 1, 2], dtype=torch.int32), + block_idx_last_scheduled_token=None, + block_idx_first_scheduled_token_p=None, + block_idx_last_computed_token=None, + block_idx_last_scheduled_token_prev_step=None, + num_computed_tokens_p=None, + seq_lens=torch.tensor([1, 1]), + ) + + # Mock KV cache (2 blocks for 2 requests) + conv_state = torch.randn((2, config.conv_L_cache - 1, dim)) + layer.kv_cache = (conv_state,) + + hidden_states = torch.randn((num_decodes, dim)) + output = torch.zeros_like(hidden_states) + + old_conv_state = conv_state.clone() + + attn_metadata_dict = {prefix: attn_metadata} + with set_forward_context(attn_metadata=attn_metadata_dict, vllm_config=vllm_config): + layer.forward_native(hidden_states, output) + + # Check if KV cache was updated + assert not torch.allclose(conv_state, old_conv_state) + + +def test_dispatch_cpu_unquantized_gemm_conv_layer(): + # Convolution layers have >2D weights; dispatch should skip them gracefully. + # Shape/dtype are AMX-pack safe (bf16, width==4, dim % block_size == 0) so + # the AMX prepack branch does not raise on AMX-capable CPUs. + class MockConvLayer(torch.nn.Module): + def __init__(self): + super().__init__() + self.weight = torch.nn.Parameter( + torch.randn(32, 1, 4, dtype=torch.bfloat16) + ) + self.bias = torch.nn.Parameter(torch.randn(32, dtype=torch.bfloat16)) + + layer = MockConvLayer() + # The ndim != 2 guard returns early without raising. + dispatch_cpu_unquantized_gemm(layer, remove_weight=False) + # No cpu_linear set — conv layers are handled elsewhere. + assert not hasattr(layer, "cpu_linear") diff --git a/vllm/model_executor/layers/mamba/short_conv.py b/vllm/model_executor/layers/mamba/short_conv.py index 79976dfff1..e7e36f2fc5 100644 --- a/vllm/model_executor/layers/mamba/short_conv.py +++ b/vllm/model_executor/layers/mamba/short_conv.py @@ -23,6 +23,7 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import ( causal_conv1d_fn, causal_conv1d_update, ) +from vllm.platforms import current_platform from vllm.utils.torch_utils import direct_register_custom_op from vllm.v1.attention.backend import AttentionMetadata from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum @@ -90,7 +91,94 @@ class ShortConv(MambaBase, CustomOp): hidden_states: torch.Tensor, output: torch.Tensor, ): - return + # Reference torch causal conv1d; runs on all CPU platforms. AMX kernels + # for causal conv can be plugged in here later. + from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import ( + causal_conv1d_torch, + causal_conv1d_update_torch, + ) + + forward_context = get_forward_context() + attn_metadata_raw = forward_context.attn_metadata + attn_metadata: AttentionMetadata | None = None + if attn_metadata_raw is not None: + assert isinstance(attn_metadata_raw, dict) + attn_metadata = attn_metadata_raw[self.prefix] + assert isinstance(attn_metadata, ShortConvAttentionMetadata) + + BCx, _ = self.in_proj(hidden_states) + B, C, x = BCx.chunk(3, dim=-1) + + # (dim, kernel_size) — same reshape as forward_cuda + conv_weights = self.conv.weight.view( + self.conv.weight.size(0), self.conv.weight.size(2) + ) + + if attn_metadata is None: + # Profile run — output value doesn't matter + Bx = (B * x).contiguous() + output_tensor, _ = self.out_proj(C * Bx) + output[: hidden_states.shape[0]] = output_tensor + return + + conv_state = ( + self.kv_cache[0] + if is_conv_state_dim_first() + else self.kv_cache[0].transpose(-1, -2) + ) # (num_blocks, dim, state_len) + + num_prefills = attn_metadata.num_prefills + num_decodes = attn_metadata.num_decode_tokens + num_prefill_tokens = attn_metadata.num_prefill_tokens + has_prefill = num_prefills > 0 + has_decode = num_decodes > 0 + num_actual_tokens = num_decodes + num_prefill_tokens + + B_d, B_p = torch.split( + B[:num_actual_tokens], [num_decodes, num_prefill_tokens], dim=0 + ) + C_d, C_p = torch.split( + C[:num_actual_tokens], [num_decodes, num_prefill_tokens], dim=0 + ) + x_d, x_p = torch.split( + x[:num_actual_tokens], [num_decodes, num_prefill_tokens], dim=0 + ) + + conv_output_list = [] + + if has_prefill: + assert attn_metadata.state_indices_tensor_p is not None + Bx_p = (B_p * x_p).transpose(0, 1) # (dim, num_prefill_tokens) + out_p = causal_conv1d_torch( + Bx_p, + conv_weights, + self.conv.bias, + conv_state, + attn_metadata.query_start_loc_p, + attn_metadata.state_indices_tensor_p.flatten(), + attn_metadata.has_initial_states_p, + activation=None, + ).transpose(0, 1)[:num_prefill_tokens] # (num_prefill_tokens, dim) + conv_output_list.append(C_p * out_p) + + if has_decode: + assert attn_metadata.state_indices_tensor_d is not None + state_indices_d = attn_metadata.state_indices_tensor_d.flatten() + Bx_d = (B_d * x_d).unsqueeze(-1) # (num_decodes, dim, 1) + # Advanced indexing returns a copy; update in-place then scatter back + gathered = conv_state[state_indices_d] # (num_decodes, dim, state_len) + out_d = causal_conv1d_update_torch( + Bx_d, + gathered, + conv_weights, + self.conv.bias, + activation=None, + ).squeeze(-1) # (num_decodes, dim) + conv_state[state_indices_d] = gathered + conv_output_list.insert(0, C_d * out_d) + + hidden_states_out = torch.vstack(conv_output_list) + output[:num_actual_tokens], _ = self.out_proj(hidden_states_out) def forward( self, @@ -235,7 +323,10 @@ def short_conv( ) -> None: forward_context: ForwardContext = get_forward_context() self = forward_context.no_compile_layers[layer_name] - self.forward_cuda(hidden_states=hidden_states, output=output) + if not current_platform.is_cpu(): + self.forward_cuda(hidden_states=hidden_states, output=output) + else: + self.forward_native(hidden_states=hidden_states, output=output) def short_conv_fake(