Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Codex <codex@openai.com>
705 lines
25 KiB
Python
705 lines
25 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
from collections.abc import Iterable
|
|
|
|
import numpy as np
|
|
import torch
|
|
import torch.nn as nn
|
|
from einops import rearrange
|
|
from transformers import PreTrainedConfig
|
|
|
|
from vllm.distributed import parallel_state
|
|
from vllm.distributed import utils as dist_utils
|
|
from vllm.model_executor.layers.activation import get_act_fn
|
|
from vllm.model_executor.layers.attention.mm_encoder_attention import (
|
|
MMEncoderAttention,
|
|
)
|
|
from vllm.model_executor.layers.conv import Conv3dLayer
|
|
from vllm.model_executor.layers.linear import (
|
|
ColumnParallelLinear,
|
|
QKVParallelLinear,
|
|
RowParallelLinear,
|
|
)
|
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
|
from vllm.model_executor.layers.rotary_embedding.mrope import triton_mrope
|
|
from vllm.model_executor.layers.rotary_embedding.mrope_vit_setup import (
|
|
vit_mrope_setup,
|
|
)
|
|
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
|
from vllm.model_executor.models.utils import maybe_prefix
|
|
from vllm.model_executor.models.vision import (
|
|
get_vit_attn_backend,
|
|
is_vit_use_data_parallel,
|
|
)
|
|
|
|
|
|
class MiniMaxVLPatchEmbed(nn.Module):
|
|
"""Conv3d-based patch embedding.
|
|
|
|
Takes flat tokens of shape (N, C * temporal_patch_size * patch_size²)
|
|
and projects each to a hidden-size embedding.
|
|
"""
|
|
|
|
def __init__(self, config: PreTrainedConfig) -> None:
|
|
super().__init__()
|
|
compression = config.img_token_compression_config
|
|
temporal_patch_size = compression.get("temporal_patch_size", 2)
|
|
patch_size = config.patch_size
|
|
num_channels = config.num_channels
|
|
|
|
self.patch_size = patch_size
|
|
self.temporal_patch_size = temporal_patch_size
|
|
self.num_channels = num_channels
|
|
self.hidden_size = config.hidden_size
|
|
|
|
self.patch_embedding = Conv3dLayer(
|
|
in_channels=num_channels,
|
|
out_channels=config.hidden_size,
|
|
kernel_size=(temporal_patch_size, patch_size, patch_size),
|
|
stride=(temporal_patch_size, patch_size, patch_size),
|
|
bias=False,
|
|
)
|
|
|
|
def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
|
|
# pixel_values: (N, C * temporal_patch_size * patch_size²)
|
|
if self.patch_embedding.weight.dtype != pixel_values.dtype:
|
|
self.patch_embedding = self.patch_embedding.to(pixel_values.dtype)
|
|
x = pixel_values.reshape(
|
|
pixel_values.shape[0],
|
|
self.num_channels,
|
|
self.temporal_patch_size,
|
|
self.patch_size,
|
|
self.patch_size,
|
|
)
|
|
return self.patch_embedding(x).reshape(x.shape[0], -1)
|
|
|
|
|
|
class MiniMaxVLAttention(nn.Module):
|
|
"""Multi-head attention with MiniMax's partial 3D RoPE.
|
|
|
|
Partial means only the first ``rot_dim`` (< head_dim) dimensions of
|
|
Q and K are rotated; the remaining dims are passed through unchanged.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
embed_dim: int,
|
|
num_heads: int,
|
|
quant_config: QuantizationConfig | None = None,
|
|
prefix: str = "",
|
|
) -> None:
|
|
super().__init__()
|
|
use_data_parallel = is_vit_use_data_parallel()
|
|
self.tp_size = (
|
|
1
|
|
if use_data_parallel
|
|
else parallel_state.get_tensor_model_parallel_world_size()
|
|
)
|
|
self.head_dim = embed_dim // num_heads
|
|
self.num_heads_per_partition = dist_utils.divide(num_heads, self.tp_size)
|
|
|
|
self.qkv_proj = QKVParallelLinear(
|
|
hidden_size=embed_dim,
|
|
head_size=self.head_dim,
|
|
total_num_heads=num_heads,
|
|
total_num_kv_heads=num_heads,
|
|
bias=True,
|
|
quant_config=quant_config,
|
|
prefix=f"{prefix}.qkv_proj",
|
|
disable_tp=use_data_parallel,
|
|
)
|
|
self.out_proj = RowParallelLinear(
|
|
input_size=embed_dim,
|
|
output_size=embed_dim,
|
|
bias=True,
|
|
quant_config=quant_config,
|
|
prefix=f"{prefix}.out_proj",
|
|
disable_tp=use_data_parallel,
|
|
)
|
|
self.attn = MMEncoderAttention(
|
|
num_heads=self.num_heads_per_partition,
|
|
head_size=self.head_dim,
|
|
prefix=f"{prefix}.attn",
|
|
)
|
|
# Partial 3D RoPE geometry: rot_dim is split evenly across t/h/w
|
|
# (same formula as MiniMaxVLVisionTransformer; rot_dim may be <
|
|
# head_dim). mrope_section is in half-dim units for triton_mrope.
|
|
rope_dims = 2 * (self.head_dim // 2)
|
|
axis_dim = 2 * ((rope_dims // 3) // 2)
|
|
self.rotary_dim = 3 * axis_dim
|
|
self.mrope_section = [axis_dim // 2] * 3
|
|
|
|
def forward(
|
|
self,
|
|
x: torch.Tensor,
|
|
cu_seqlens: torch.Tensor,
|
|
rotary_cos: torch.Tensor,
|
|
rotary_sin: torch.Tensor,
|
|
max_seqlen: torch.Tensor,
|
|
sequence_lengths: torch.Tensor | None = None,
|
|
) -> torch.Tensor:
|
|
# x: (N, 1, embed_dim) [seq=N, batch=1, chan=embed_dim]
|
|
x_qkv, _ = self.qkv_proj(x) # (N, 1, 3 * heads_per_part * head_dim)
|
|
seq_len, batch_size, _ = x_qkv.shape
|
|
|
|
qkv = x_qkv.view(seq_len, 3, self.num_heads_per_partition, self.head_dim)
|
|
v = qkv[:, 2].unsqueeze(0) # (b=1, N, heads, head_dim) strided view
|
|
|
|
# In-place partial 3D RoPE on strided q/k views of the packed qkv
|
|
# tensor: one kernel launch, no repack/upcast copies. fp32 cos/sin
|
|
# make the kernel compute in fp32, matching the reference
|
|
# ``_minimax_rope_applier`` precision.
|
|
q = qkv[:, 0].reshape(seq_len, -1)
|
|
k = qkv[:, 1].reshape(seq_len, -1)
|
|
triton_mrope(
|
|
q,
|
|
k,
|
|
rotary_cos,
|
|
rotary_sin,
|
|
self.mrope_section,
|
|
self.head_dim,
|
|
self.rotary_dim,
|
|
mrope_interleaved=False,
|
|
is_neox_style=True,
|
|
)
|
|
q = q.view(batch_size, seq_len, self.num_heads_per_partition, self.head_dim)
|
|
k = k.view(batch_size, seq_len, self.num_heads_per_partition, self.head_dim)
|
|
|
|
# Flash attention → (b, N, heads, head_dim)
|
|
context = self.attn(
|
|
query=q,
|
|
key=k,
|
|
value=v,
|
|
cu_seqlens=cu_seqlens,
|
|
max_seqlen=max_seqlen,
|
|
sequence_lengths=sequence_lengths,
|
|
)
|
|
|
|
# Back to (N, 1, embed_dim)
|
|
context = rearrange(context, "b s h d -> s b (h d)", b=batch_size)
|
|
output, _ = self.out_proj(context)
|
|
return output
|
|
|
|
|
|
class MiniMaxVLEncoderLayer(nn.Module):
|
|
"""Single CLIP-style transformer block."""
|
|
|
|
def __init__(
|
|
self,
|
|
config: PreTrainedConfig,
|
|
quant_config: QuantizationConfig | None = None,
|
|
prefix: str = "",
|
|
) -> None:
|
|
super().__init__()
|
|
embed_dim = config.hidden_size
|
|
self.layer_norm1 = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
|
|
self.self_attn = MiniMaxVLAttention(
|
|
embed_dim=embed_dim,
|
|
num_heads=config.num_attention_heads,
|
|
quant_config=quant_config,
|
|
prefix=f"{prefix}.self_attn",
|
|
)
|
|
self.layer_norm2 = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
|
|
use_data_parallel = is_vit_use_data_parallel()
|
|
self.fc1 = ColumnParallelLinear(
|
|
config.hidden_size,
|
|
config.intermediate_size,
|
|
bias=True,
|
|
quant_config=quant_config,
|
|
prefix=f"{prefix}.fc1",
|
|
disable_tp=use_data_parallel,
|
|
)
|
|
self.act = get_act_fn(getattr(config, "hidden_act", "gelu"))
|
|
self.fc2 = RowParallelLinear(
|
|
config.intermediate_size,
|
|
config.hidden_size,
|
|
bias=True,
|
|
quant_config=quant_config,
|
|
prefix=f"{prefix}.fc2",
|
|
disable_tp=use_data_parallel,
|
|
)
|
|
|
|
def forward(
|
|
self,
|
|
x: torch.Tensor,
|
|
cu_seqlens: torch.Tensor,
|
|
rotary_cos: torch.Tensor,
|
|
rotary_sin: torch.Tensor,
|
|
max_seqlen: torch.Tensor,
|
|
sequence_lengths: torch.Tensor | None = None,
|
|
) -> torch.Tensor:
|
|
# x: (N, 1, hidden_size)
|
|
x = x + self.self_attn(
|
|
self.layer_norm1(x),
|
|
cu_seqlens,
|
|
rotary_cos,
|
|
rotary_sin,
|
|
max_seqlen,
|
|
sequence_lengths,
|
|
)
|
|
residual = x
|
|
x, _ = self.fc1(self.layer_norm2(x))
|
|
x = self.act(x)
|
|
x, _ = self.fc2(x)
|
|
return residual + x
|
|
|
|
|
|
class MiniMaxVLEncoder(nn.Module):
|
|
def __init__(
|
|
self,
|
|
config: PreTrainedConfig,
|
|
num_hidden_layers_override: int | None = None,
|
|
quant_config: QuantizationConfig | None = None,
|
|
prefix: str = "",
|
|
) -> None:
|
|
super().__init__()
|
|
n = (
|
|
config.num_hidden_layers
|
|
if num_hidden_layers_override is None
|
|
else num_hidden_layers_override
|
|
)
|
|
self.layers = nn.ModuleList(
|
|
[
|
|
MiniMaxVLEncoderLayer(
|
|
config=config,
|
|
quant_config=quant_config,
|
|
prefix=f"{prefix}.layers.{i}",
|
|
)
|
|
for i in range(n)
|
|
]
|
|
)
|
|
|
|
def forward(
|
|
self,
|
|
x: torch.Tensor,
|
|
cu_seqlens: torch.Tensor,
|
|
rotary_cos: torch.Tensor,
|
|
rotary_sin: torch.Tensor,
|
|
max_seqlen: torch.Tensor,
|
|
sequence_lengths: torch.Tensor | None = None,
|
|
) -> torch.Tensor:
|
|
for layer in self.layers:
|
|
x = layer(
|
|
x,
|
|
cu_seqlens,
|
|
rotary_cos,
|
|
rotary_sin,
|
|
max_seqlen,
|
|
sequence_lengths,
|
|
)
|
|
return x
|
|
|
|
|
|
class MiniMaxVLVisionTransformer(nn.Module):
|
|
"""CLIP-based ViT with 3D RoPE (t/h/w decomposed).
|
|
|
|
Faithfully mirrors the reference ``MiniMaxVLVisionTransformer``.
|
|
FLASHINFER backend is not supported; standard flash-attn is used.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
config: PreTrainedConfig,
|
|
num_hidden_layers_override: int | None = None,
|
|
require_post_norm: bool | None = None,
|
|
quant_config: QuantizationConfig | None = None,
|
|
prefix: str = "",
|
|
) -> None:
|
|
super().__init__()
|
|
compression = config.img_token_compression_config
|
|
self.spatial_merge_size: int = compression.get("spatial_merge_size", 2)
|
|
self.temporal_patch_size: int = compression.get("temporal_patch_size", 2)
|
|
self.vision_segment_max_frames: int | None = getattr(
|
|
config, "vision_segment_max_frames", None
|
|
)
|
|
self.use_data_parallel = is_vit_use_data_parallel()
|
|
|
|
embed_dim = config.hidden_size
|
|
head_dim = embed_dim // config.num_attention_heads
|
|
# Backend selection + sharding info for building encoder metadata.
|
|
# Defaults to FLASH_ATTN on SM80+; --mm-encoder-attn-backend FLASHINFER
|
|
# selects the cuDNN ViT prefill path.
|
|
self.hidden_size = embed_dim
|
|
self.tp_size = (
|
|
1
|
|
if self.use_data_parallel
|
|
else parallel_state.get_tensor_model_parallel_world_size()
|
|
)
|
|
self.attn_backend = get_vit_attn_backend(
|
|
head_size=head_dim, dtype=torch.get_default_dtype()
|
|
)
|
|
rope_dims = 2 * (head_dim // 2)
|
|
|
|
# Split rope dims evenly across t/h/w (same formula as the reference)
|
|
self.t_dim = int(2 * ((rope_dims // 3) // 2))
|
|
self.h_dim = int(2 * ((rope_dims // 3) // 2))
|
|
self.w_dim = int(2 * ((rope_dims // 3) // 2))
|
|
# rot_dim = t_dim + h_dim + w_dim (may be < head_dim)
|
|
|
|
rope_theta: float = getattr(config, "rope_theta", 10000.0)
|
|
inv_freq_t = 1.0 / (
|
|
rope_theta
|
|
** (torch.arange(0, self.t_dim, 2, dtype=torch.float32) / self.t_dim)
|
|
)
|
|
inv_freq_h = 1.0 / (
|
|
rope_theta
|
|
** (torch.arange(0, self.h_dim, 2, dtype=torch.float32) / self.h_dim)
|
|
)
|
|
inv_freq_w = 1.0 / (
|
|
rope_theta
|
|
** (torch.arange(0, self.w_dim, 2, dtype=torch.float32) / self.w_dim)
|
|
)
|
|
self.register_buffer("inv_freq_t", inv_freq_t, persistent=False)
|
|
self.register_buffer("inv_freq_h", inv_freq_h, persistent=False)
|
|
self.register_buffer("inv_freq_w", inv_freq_w, persistent=False)
|
|
|
|
self.embeddings = MiniMaxVLPatchEmbed(config)
|
|
self.pre_layrnorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
|
|
|
|
n_layers = config.num_hidden_layers
|
|
if num_hidden_layers_override is None:
|
|
num_hidden_layers_override = n_layers
|
|
self.encoder = MiniMaxVLEncoder(
|
|
config=config,
|
|
num_hidden_layers_override=num_hidden_layers_override,
|
|
quant_config=quant_config,
|
|
prefix=f"{prefix}.encoder",
|
|
)
|
|
|
|
if require_post_norm is None:
|
|
require_post_norm = num_hidden_layers_override == n_layers
|
|
self.post_layernorm = (
|
|
nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
|
|
if require_post_norm
|
|
else None
|
|
)
|
|
|
|
# out_hidden_size needed by run_dp_sharded_mrope_vision_model
|
|
self.out_hidden_size = embed_dim
|
|
|
|
# ── Frame-limit helper (mirrors the reference) ───────────────────────
|
|
|
|
def _apply_max_frames_limit(self, grid_thw: list[list[int]]) -> list[list[int]]:
|
|
if self.vision_segment_max_frames is None:
|
|
return grid_thw
|
|
max_f = self.vision_segment_max_frames
|
|
out: list[list[int]] = []
|
|
for t, h, w in grid_thw:
|
|
if t <= max_f:
|
|
out.append([t, h, w])
|
|
else:
|
|
for i in range(0, t, max_f):
|
|
out.append([min(max_f, t - i), h, w])
|
|
return out
|
|
|
|
# ── Encoder metadata (shared by eager forward and CUDA graph replay) ──
|
|
|
|
def prepare_encoder_metadata(
|
|
self,
|
|
grid_thw: list[list[int]],
|
|
*,
|
|
device: torch.device,
|
|
max_batch_size: int | None = None,
|
|
max_seqlen_override: int | None = None,
|
|
) -> dict[str, torch.Tensor | None]:
|
|
"""Build all grid-dependent forward inputs as standalone tensors.
|
|
|
|
The eager path calls this per forward; encoder CUDA graphs call it at
|
|
capture (dummy grids, padded to fixed buffer sizes) and before each
|
|
replay (real grids, unpadded — the manager's padding logic completes
|
|
the captured buffers).
|
|
|
|
Args:
|
|
grid_thw: per-item [t, h, w] patch grids (host metadata).
|
|
device: device for the attention metadata tensors.
|
|
max_batch_size: pad cu_seqlens to this many segments with empty
|
|
(zero-length) trailing segments; None leaves it unpadded.
|
|
max_seqlen_override: bake this max_seqlen instead of deriving it
|
|
from the grids; CUDA graph capture passes a worst-case value
|
|
because max_seqlen is a CPU scalar baked into the graph.
|
|
|
|
"""
|
|
limited = self._apply_max_frames_limit(grid_thw)
|
|
|
|
# Token-level cumulative sequence lengths (one segment per limited grid).
|
|
lens = [t * h * w for t, h, w in limited]
|
|
cu_seqlens_np = np.zeros(len(lens) + 1, dtype=np.int32)
|
|
np.cumsum(np.array(lens, dtype=np.int32), out=cu_seqlens_np[1:])
|
|
|
|
if max_batch_size is not None and len(lens) < max_batch_size:
|
|
pad = np.full(max_batch_size - len(lens), cu_seqlens_np[-1], dtype=np.int32)
|
|
cu_seqlens_np = np.concatenate([cu_seqlens_np, pad])
|
|
|
|
# Backend-specific encoder metadata. For FLASH_ATTN this returns the raw
|
|
# token cu_seqlens, the max segment length, and sequence_lengths=None;
|
|
# for FLASHINFER (cuDNN) it repacks cu_seqlens into element-offset
|
|
# indptrs, buckets max_seqlen, and builds padded per-sequence lengths.
|
|
sequence_lengths = MMEncoderAttention.maybe_compute_seq_lens(
|
|
self.attn_backend, cu_seqlens_np, device
|
|
)
|
|
max_seqlen = torch.tensor(
|
|
max_seqlen_override
|
|
if max_seqlen_override is not None
|
|
else MMEncoderAttention.compute_max_seqlen(
|
|
self.attn_backend, cu_seqlens_np
|
|
),
|
|
dtype=torch.int32,
|
|
)
|
|
cu_seqlens = MMEncoderAttention.maybe_recompute_cu_seqlens(
|
|
self.attn_backend,
|
|
cu_seqlens_np,
|
|
self.hidden_size,
|
|
self.tp_size,
|
|
device,
|
|
)
|
|
|
|
# 3D RoPE cos/sin: (3, total_N, half_rot_dim) fp32 t/h/w planes for
|
|
# triton_mrope from one fused setup kernel. Kept fp32 so the kernel
|
|
# computes the rotation in fp32, matching the reference.
|
|
rotary_cos, rotary_sin = vit_mrope_setup(
|
|
self.inv_freq_t,
|
|
self.inv_freq_h,
|
|
self.inv_freq_w,
|
|
limited,
|
|
self.spatial_merge_size,
|
|
)
|
|
|
|
return {
|
|
"rotary_cos": rotary_cos,
|
|
"rotary_sin": rotary_sin,
|
|
"cu_seqlens": cu_seqlens,
|
|
"max_seqlen": max_seqlen,
|
|
"sequence_lengths": sequence_lengths,
|
|
}
|
|
|
|
# ── Forward ──────────────────────────────────────────────────────────
|
|
|
|
def forward(
|
|
self,
|
|
pixel_values: torch.Tensor,
|
|
grid_thw: list[list[int]] | None = None,
|
|
*,
|
|
encoder_metadata: dict[str, torch.Tensor | None] | None = None,
|
|
) -> torch.Tensor:
|
|
# pixel_values: (total_N, C * temporal_patch_size * patch_size²)
|
|
# Output: (total_N, hidden_size)
|
|
|
|
hidden = self.embeddings(pixel_values) # (total_N, hidden_size)
|
|
hidden = self.pre_layrnorm(hidden)
|
|
|
|
if encoder_metadata is None:
|
|
assert grid_thw is not None
|
|
encoder_metadata = self.prepare_encoder_metadata(
|
|
grid_thw, device=hidden.device
|
|
)
|
|
|
|
cu_seqlens = encoder_metadata["cu_seqlens"]
|
|
rotary_cos = encoder_metadata["rotary_cos"]
|
|
rotary_sin = encoder_metadata["rotary_sin"]
|
|
max_seqlen = encoder_metadata["max_seqlen"]
|
|
sequence_lengths = encoder_metadata.get("sequence_lengths")
|
|
assert cu_seqlens is not None
|
|
assert rotary_cos is not None
|
|
assert rotary_sin is not None
|
|
assert max_seqlen is not None
|
|
|
|
# Encoder expects (N, 1, hidden_size) — add batch dim
|
|
hidden = hidden.unsqueeze(1)
|
|
|
|
hidden = self.encoder(
|
|
hidden,
|
|
cu_seqlens,
|
|
rotary_cos,
|
|
rotary_sin,
|
|
max_seqlen,
|
|
sequence_lengths=sequence_lengths,
|
|
)
|
|
hidden = hidden.squeeze(1) # back to (total_N, hidden_size)
|
|
|
|
if self.post_layernorm is not None:
|
|
hidden = self.post_layernorm(hidden)
|
|
|
|
return hidden
|
|
|
|
|
|
class MiniMaxVLMultiModalProjector(nn.Module):
|
|
"""Two-layer MLP projector: vision_hidden → text_hidden."""
|
|
|
|
def __init__(
|
|
self,
|
|
vision_hidden_size: int,
|
|
text_hidden_size: int,
|
|
projector_hidden_size: int | None,
|
|
multimodal_projector_bias: bool,
|
|
projector_hidden_act: str = "gelu",
|
|
quant_config: QuantizationConfig | None = None,
|
|
prefix: str = "",
|
|
) -> None:
|
|
super().__init__()
|
|
mid = projector_hidden_size if projector_hidden_size else text_hidden_size
|
|
use_dp = is_vit_use_data_parallel()
|
|
self.linear_1 = ColumnParallelLinear(
|
|
vision_hidden_size,
|
|
mid,
|
|
bias=multimodal_projector_bias,
|
|
quant_config=quant_config,
|
|
prefix=f"{prefix}.linear_1",
|
|
disable_tp=use_dp,
|
|
)
|
|
self.act = get_act_fn(projector_hidden_act)
|
|
self.linear_2 = RowParallelLinear(
|
|
mid,
|
|
text_hidden_size,
|
|
bias=multimodal_projector_bias,
|
|
quant_config=quant_config,
|
|
prefix=f"{prefix}.linear_2",
|
|
disable_tp=use_dp,
|
|
)
|
|
|
|
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
|
x, _ = self.linear_1(x)
|
|
x = self.act(x)
|
|
x, _ = self.linear_2(x)
|
|
return x
|
|
|
|
|
|
class MiniMaxVLPatchMerger(nn.Module):
|
|
def __init__(
|
|
self,
|
|
spatial_merge_size: int,
|
|
text_hidden_size: int,
|
|
projector_hidden_size: int | None,
|
|
patch_merge_bias: bool,
|
|
projector_hidden_act: str = "gelu",
|
|
quant_config: QuantizationConfig | None = None,
|
|
prefix: str = "",
|
|
) -> None:
|
|
super().__init__()
|
|
self.spatial_merge_size = spatial_merge_size
|
|
mid = projector_hidden_size if projector_hidden_size else text_hidden_size
|
|
merge_in = text_hidden_size * spatial_merge_size**2
|
|
use_dp = is_vit_use_data_parallel()
|
|
self.linear_1 = ColumnParallelLinear(
|
|
merge_in,
|
|
mid,
|
|
bias=patch_merge_bias,
|
|
quant_config=quant_config,
|
|
prefix=f"{prefix}.linear_1",
|
|
disable_tp=use_dp,
|
|
)
|
|
self.act = get_act_fn(projector_hidden_act)
|
|
self.linear_2 = RowParallelLinear(
|
|
mid,
|
|
text_hidden_size,
|
|
bias=patch_merge_bias,
|
|
quant_config=quant_config,
|
|
prefix=f"{prefix}.linear_2",
|
|
disable_tp=use_dp,
|
|
)
|
|
|
|
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
|
# x: (N, text_hidden_size) → (N // merge_size², text_hidden_size)
|
|
x = x.reshape(x.shape[0] // (self.spatial_merge_size**2), -1)
|
|
x, _ = self.linear_1(x)
|
|
x = self.act(x)
|
|
x, _ = self.linear_2(x)
|
|
return x
|
|
|
|
|
|
class MiniMaxVLVisionModel(nn.Module):
|
|
"""Full vision model: ViT → projector → patch merger."""
|
|
|
|
def __init__(
|
|
self,
|
|
config: PreTrainedConfig,
|
|
text_hidden_size: int,
|
|
projector_hidden_size: int | None = None,
|
|
quant_config: QuantizationConfig | None = None,
|
|
prefix: str = "",
|
|
) -> None:
|
|
super().__init__()
|
|
compression = config.img_token_compression_config
|
|
spatial_merge_size: int = compression.get("spatial_merge_size", 2)
|
|
self.spatial_merge_size = spatial_merge_size
|
|
self.use_data_parallel = is_vit_use_data_parallel()
|
|
|
|
# The released checkpoint ships no ``post_layernorm`` weights and
|
|
# uses ``vision_feature_layer=-1`` with ``vision_feature_select_strategy
|
|
# ="full"``, i.e. the raw last encoder hidden state (CLIP's
|
|
# ``last_hidden_state`` is taken before the post layernorm). Applying an
|
|
# untrained post layernorm here would corrupt the visual features.
|
|
self.vision_model = MiniMaxVLVisionTransformer(
|
|
config=config,
|
|
require_post_norm=False,
|
|
quant_config=quant_config,
|
|
prefix=maybe_prefix(prefix, "vision_model"),
|
|
)
|
|
self.multi_modal_projector = MiniMaxVLMultiModalProjector(
|
|
vision_hidden_size=config.hidden_size,
|
|
text_hidden_size=text_hidden_size,
|
|
projector_hidden_size=projector_hidden_size,
|
|
multimodal_projector_bias=getattr(
|
|
config, "multimodal_projector_bias", True
|
|
),
|
|
projector_hidden_act=getattr(config, "projector_hidden_act", "gelu"),
|
|
quant_config=quant_config,
|
|
prefix=maybe_prefix(prefix, "multi_modal_projector"),
|
|
)
|
|
self.patch_merge_mlp = MiniMaxVLPatchMerger(
|
|
spatial_merge_size=spatial_merge_size,
|
|
text_hidden_size=text_hidden_size,
|
|
projector_hidden_size=projector_hidden_size,
|
|
patch_merge_bias=getattr(config, "patch_merge_bias", True),
|
|
projector_hidden_act=getattr(config, "projector_hidden_act", "gelu"),
|
|
quant_config=quant_config,
|
|
prefix=maybe_prefix(prefix, "patch_merge_mlp"),
|
|
)
|
|
|
|
self.dtype = self.vision_model.embeddings.patch_embedding.weight.dtype
|
|
self.out_hidden_size = text_hidden_size
|
|
|
|
def forward(
|
|
self,
|
|
pixel_values: torch.Tensor,
|
|
grid_thw: list[list[int]] | None = None,
|
|
*,
|
|
encoder_metadata: dict[str, torch.Tensor | None] | None = None,
|
|
) -> torch.Tensor:
|
|
hidden = self.vision_model(
|
|
pixel_values=pixel_values,
|
|
grid_thw=grid_thw,
|
|
encoder_metadata=encoder_metadata,
|
|
)
|
|
if hidden.dim() == 3:
|
|
hidden = hidden.squeeze(0)
|
|
hidden = self.multi_modal_projector(hidden)
|
|
hidden = self.patch_merge_mlp(hidden)
|
|
return hidden
|
|
|
|
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
|
stacked_params_mapping = [
|
|
# (param_name, shard_name, shard_id)
|
|
("qkv_proj.", "q_proj.", "q"),
|
|
("qkv_proj.", "k_proj.", "k"),
|
|
("qkv_proj.", "v_proj.", "v"),
|
|
]
|
|
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
|
loaded_params: set[str] = set()
|
|
|
|
for name, loaded_weight in weights:
|
|
for param_name, weight_name, shard_id in stacked_params_mapping:
|
|
if weight_name not in name:
|
|
continue
|
|
name = name.replace(weight_name, param_name)
|
|
|
|
param = params_dict[name]
|
|
weight_loader = param.weight_loader
|
|
weight_loader(param, loaded_weight, shard_id)
|
|
break
|
|
else:
|
|
param = params_dict[name]
|
|
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
|
weight_loader(param, loaded_weight)
|
|
loaded_params.add(name)
|
|
return loaded_params
|