From 00b7847c8036b667742b4efb21aab1de51fd4721 Mon Sep 17 00:00:00 2001 From: Thien Tran Date: Thu, 24 Sep 2026 15:41:36 +0800 Subject: [PATCH] [Perf] Use Conv3dLayer for MiniMax M3 patch embedding (#58512) Signed-off-by: OpenAI Codex Co-authored-by: OpenAI Codex --- tests/model_executor/layers/test_conv.py | 43 +++++++++++++++++++ vllm/models/minimax_m3/common/vision_tower.py | 3 +- 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 tests/model_executor/layers/test_conv.py diff --git a/tests/model_executor/layers/test_conv.py b/tests/model_executor/layers/test_conv.py new file mode 100644 index 0000000000..f7b23fbdcd --- /dev/null +++ b/tests/model_executor/layers/test_conv.py @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch +from torch import nn + +from vllm.model_executor.layers.conv import Conv3dLayer + + +@pytest.mark.parametrize( + "input_shape,kernel_size,out_channels,bias", + [ + pytest.param((4, 3, 2, 14, 14), (2, 14, 14), 1280, False, id="minimax_m3"), + pytest.param((4, 3, 2, 14, 14), (2, 14, 14), 1152, False, id="qwen2_vl"), + pytest.param((4, 3, 2, 14, 14), (2, 14, 14), 1152, True, id="qwen3_vl"), + pytest.param((4, 3, 1, 14, 14), (1, 14, 14), 1536, True, id="glm_vl"), + pytest.param((1, 3, 4, 28, 28), (2, 14, 14), 32, True, id="multiple_patches"), + ], +) +def test_conv3d_patch_embedding_correctness( + default_vllm_config, input_shape, kernel_size, out_channels, bias +): + device = "cuda" if torch.cuda.is_available() else "cpu" + dtype = torch.bfloat16 if device == "cuda" else torch.float32 + reference = nn.Conv3d( + input_shape[1], out_channels, kernel_size, stride=kernel_size, bias=bias + ).to(device=device, dtype=dtype) + layer = Conv3dLayer( + input_shape[1], + out_channels, + kernel_size, + stride=kernel_size, + bias=bias, + params_dtype=dtype, + ).to(device) + layer.load_state_dict(reference.state_dict(), strict=True) + assert layer.enable_linear + + x = torch.randn(input_shape, device=device, dtype=dtype) + expected = reference(x) + atol = rtol = 0.02 if dtype == torch.bfloat16 else 1e-5 + torch.testing.assert_close(layer(x), expected, atol=atol, rtol=rtol) diff --git a/vllm/models/minimax_m3/common/vision_tower.py b/vllm/models/minimax_m3/common/vision_tower.py index 23b8b3ed31..44a0964f7b 100644 --- a/vllm/models/minimax_m3/common/vision_tower.py +++ b/vllm/models/minimax_m3/common/vision_tower.py @@ -14,6 +14,7 @@ from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.layers.attention.mm_encoder_attention import ( MMEncoderAttention, ) +from vllm.model_executor.layers.conv import Conv3dLayer from vllm.model_executor.layers.linear import ( ColumnParallelLinear, QKVParallelLinear, @@ -55,7 +56,7 @@ class MiniMaxVLPatchEmbed(nn.Module): self.num_channels = num_channels self.hidden_size = config.hidden_size - self.patch_embedding = nn.Conv3d( + self.patch_embedding = Conv3dLayer( in_channels=num_channels, out_channels=config.hidden_size, kernel_size=(temporal_patch_size, patch_size, patch_size),