[Perf] Use Conv3dLayer for MiniMax M3 patch embedding (#58512)
Signed-off-by: OpenAI Codex <codex@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
1291bdb85e
commit
00b7847c80
@@ -0,0 +1,43 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.model_executor.layers.conv import Conv3dLayer
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"input_shape,kernel_size,out_channels,bias",
|
||||
[
|
||||
pytest.param((4, 3, 2, 14, 14), (2, 14, 14), 1280, False, id="minimax_m3"),
|
||||
pytest.param((4, 3, 2, 14, 14), (2, 14, 14), 1152, False, id="qwen2_vl"),
|
||||
pytest.param((4, 3, 2, 14, 14), (2, 14, 14), 1152, True, id="qwen3_vl"),
|
||||
pytest.param((4, 3, 1, 14, 14), (1, 14, 14), 1536, True, id="glm_vl"),
|
||||
pytest.param((1, 3, 4, 28, 28), (2, 14, 14), 32, True, id="multiple_patches"),
|
||||
],
|
||||
)
|
||||
def test_conv3d_patch_embedding_correctness(
|
||||
default_vllm_config, input_shape, kernel_size, out_channels, bias
|
||||
):
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
dtype = torch.bfloat16 if device == "cuda" else torch.float32
|
||||
reference = nn.Conv3d(
|
||||
input_shape[1], out_channels, kernel_size, stride=kernel_size, bias=bias
|
||||
).to(device=device, dtype=dtype)
|
||||
layer = Conv3dLayer(
|
||||
input_shape[1],
|
||||
out_channels,
|
||||
kernel_size,
|
||||
stride=kernel_size,
|
||||
bias=bias,
|
||||
params_dtype=dtype,
|
||||
).to(device)
|
||||
layer.load_state_dict(reference.state_dict(), strict=True)
|
||||
assert layer.enable_linear
|
||||
|
||||
x = torch.randn(input_shape, device=device, dtype=dtype)
|
||||
expected = reference(x)
|
||||
atol = rtol = 0.02 if dtype == torch.bfloat16 else 1e-5
|
||||
torch.testing.assert_close(layer(x), expected, atol=atol, rtol=rtol)
|
||||
@@ -14,6 +14,7 @@ from vllm.model_executor.layers.activation import get_act_fn
|
||||
from vllm.model_executor.layers.attention.mm_encoder_attention import (
|
||||
MMEncoderAttention,
|
||||
)
|
||||
from vllm.model_executor.layers.conv import Conv3dLayer
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
@@ -55,7 +56,7 @@ class MiniMaxVLPatchEmbed(nn.Module):
|
||||
self.num_channels = num_channels
|
||||
self.hidden_size = config.hidden_size
|
||||
|
||||
self.patch_embedding = nn.Conv3d(
|
||||
self.patch_embedding = Conv3dLayer(
|
||||
in_channels=num_channels,
|
||||
out_channels=config.hidden_size,
|
||||
kernel_size=(temporal_patch_size, patch_size, patch_size),
|
||||
|
||||
Reference in New Issue
Block a user