[Model] Add Bailing V3 VL support (#55921)

Signed-off-by: zexplorerhj <zhjoneson@163.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
zexplorerhj
2026-09-10 02:46:56 -07:00
committed by GitHub
co-authored by Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 6ff479e1f7
commit fe26c705f7
19 changed files with 1825 additions and 20 deletions
@@ -6,6 +6,7 @@ from typing import Any
import torch
from .bailing_mrope import BailingMRotaryEmbedding
from .base import RotaryEmbedding
from .deepseek_scaling_rope import (
DeepseekScalingRotaryEmbedding,
@@ -97,6 +98,16 @@ def get_rope(
dtype,
**extra_kwargs,
)
elif scaling_type == "bailing_mrope":
rotary_emb = BailingMRotaryEmbedding(
head_size,
rotary_dim,
max_position,
base,
is_neox_style,
dtype,
mrope_section=rope_parameters["mrope_section"],
)
elif scaling_type == "default":
if "mrope_section" in rope_parameters:
rotary_emb = MRotaryEmbedding(
@@ -0,0 +1,351 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.triton_utils import tl, triton
from .mrope import MRotaryEmbedding
def _validate_bailing_mrope_config(
head_size: int,
rotary_dim: int,
mrope_section: list[int],
) -> int:
if rotary_dim <= 0 or rotary_dim % 2 != 0 or rotary_dim > head_size:
raise ValueError(
"Bailing M-RoPE requires an even rotary_dim in (0, head_size], "
f"got rotary_dim={rotary_dim} and head_size={head_size}"
)
if len(mrope_section) != 3:
raise ValueError(
"Bailing M-RoPE expects [T, H, W] sections, "
f"got mrope_section={mrope_section}"
)
temporal_size, height_size, width_size = mrope_section
if any(section_size < 0 for section_size in mrope_section):
raise ValueError(
"Bailing M-RoPE sections must be non-negative, "
f"got mrope_section={mrope_section}"
)
if height_size != width_size:
raise ValueError(
"Bailing M-RoPE requires equal H and W sections, "
f"got height_size={height_size} and width_size={width_size}"
)
spatial_size = height_size + width_size
if spatial_size + temporal_size != rotary_dim // 2:
raise ValueError(
"Bailing M-RoPE sections must sum to rotary_dim // 2, got "
f"mrope_section={mrope_section} and rotary_dim={rotary_dim}"
)
return spatial_size
def _validate_bailing_positions(
positions: torch.Tensor,
*,
require_2d: bool = False,
) -> None:
if positions.dtype not in (torch.int32, torch.int64):
raise ValueError(
"Bailing M-RoPE positions must have int32 or int64 dtype, got "
f"dtype={positions.dtype}"
)
if positions.ndim == 1 and not require_2d:
return
if positions.ndim == 2:
if positions.shape[0] == 3:
return
raise ValueError(
"Bailing M-RoPE expects T/H/W positions with shape [3, N], "
f"got shape={tuple(positions.shape)}"
)
if require_2d:
raise ValueError(
"Bailing M-RoPE expects T/H/W positions with shape [3, N], "
f"got shape={tuple(positions.shape)}"
)
raise ValueError(
f"Bailing M-RoPE positions must be 1D or 2D, got shape={tuple(positions.shape)}"
)
@triton.jit
def _triton_bailing_mrope_forward(
q_ptr,
k_ptr,
positions_ptr,
cos_sin_cache_ptr,
q_stride_token,
q_stride_head,
q_stride_dim,
k_stride_token,
k_stride_head,
k_stride_dim,
positions_stride_axis,
positions_stride_token,
cache_stride_position,
cache_stride_dim,
n_qh: tl.constexpr,
n_kh: tl.constexpr,
rotary_dim: tl.constexpr,
pad_n_qh: tl.constexpr,
pad_n_kh: tl.constexpr,
pad_half_rotary_dim: tl.constexpr,
spatial_size: tl.constexpr,
):
"""Apply GPT-J-style H/W-interleaved, T-tail M-RoPE in place."""
token_idx = tl.program_id(0)
position_offset = token_idx * positions_stride_token
temporal_position = tl.load(positions_ptr + position_offset).to(tl.int64)
height_position = tl.load(
positions_ptr + positions_stride_axis + position_offset
).to(tl.int64)
width_position = tl.load(
positions_ptr + 2 * positions_stride_axis + position_offset
).to(tl.int64)
frequency_offsets = tl.arange(0, pad_half_rotary_dim)
frequency_mask = frequency_offsets < rotary_dim // 2
spatial_position = tl.where(
frequency_offsets % 2 == 0,
height_position,
width_position,
)
selected_position = tl.where(
frequency_offsets < spatial_size,
spatial_position,
temporal_position,
)
cache_offsets = (
selected_position * cache_stride_position + frequency_offsets * cache_stride_dim
)
cos_row = tl.load(
cos_sin_cache_ptr + cache_offsets,
mask=frequency_mask,
other=0.0,
).to(tl.float32)
sin_row = tl.load(
cos_sin_cache_ptr + cache_offsets + (rotary_dim // 2) * cache_stride_dim,
mask=frequency_mask,
other=0.0,
).to(tl.float32)
q_head_offsets = tl.arange(0, pad_n_qh)[:, None]
k_head_offsets = tl.arange(0, pad_n_kh)[:, None]
pair_offsets = frequency_offsets[None, :]
q_mask = (q_head_offsets < n_qh) & frequency_mask[None, :]
k_mask = (k_head_offsets < n_kh) & frequency_mask[None, :]
q_base = q_ptr + token_idx * q_stride_token
q_even_offsets = q_head_offsets * q_stride_head + 2 * pair_offsets * q_stride_dim
q_odd_offsets = q_even_offsets + q_stride_dim
q_even = tl.load(q_base + q_even_offsets, mask=q_mask, other=0.0).to(tl.float32)
q_odd = tl.load(q_base + q_odd_offsets, mask=q_mask, other=0.0).to(tl.float32)
q_even_cos = (q_even * cos_row[None, :]).to(q_ptr.type.element_ty).to(tl.float32)
q_odd_sin = (q_odd * sin_row[None, :]).to(q_ptr.type.element_ty).to(tl.float32)
q_odd_cos = (q_odd * cos_row[None, :]).to(q_ptr.type.element_ty).to(tl.float32)
q_even_sin = (q_even * sin_row[None, :]).to(q_ptr.type.element_ty).to(tl.float32)
tl.store(
q_base + q_even_offsets,
q_even_cos - q_odd_sin,
mask=q_mask,
)
tl.store(
q_base + q_odd_offsets,
q_odd_cos + q_even_sin,
mask=q_mask,
)
k_base = k_ptr + token_idx * k_stride_token
k_even_offsets = k_head_offsets * k_stride_head + 2 * pair_offsets * k_stride_dim
k_odd_offsets = k_even_offsets + k_stride_dim
k_even = tl.load(k_base + k_even_offsets, mask=k_mask, other=0.0).to(tl.float32)
k_odd = tl.load(k_base + k_odd_offsets, mask=k_mask, other=0.0).to(tl.float32)
k_even_cos = (k_even * cos_row[None, :]).to(k_ptr.type.element_ty).to(tl.float32)
k_odd_sin = (k_odd * sin_row[None, :]).to(k_ptr.type.element_ty).to(tl.float32)
k_odd_cos = (k_odd * cos_row[None, :]).to(k_ptr.type.element_ty).to(tl.float32)
k_even_sin = (k_even * sin_row[None, :]).to(k_ptr.type.element_ty).to(tl.float32)
tl.store(
k_base + k_even_offsets,
k_even_cos - k_odd_sin,
mask=k_mask,
)
tl.store(
k_base + k_odd_offsets,
k_odd_cos + k_even_sin,
mask=k_mask,
)
def triton_bailing_mrope(
query: torch.Tensor,
key: torch.Tensor,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
spatial_size: int,
head_size: int,
rotary_dim: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Fused Bailing M-RoPE for packed or strided Q/K."""
_validate_bailing_positions(positions, require_2d=True)
num_tokens = positions.shape[-1]
if num_tokens == 0:
return query, key
query_3d = query.view(num_tokens, -1, head_size)
key_3d = key.view(num_tokens, -1, head_size)
n_q_heads = query_3d.shape[1]
n_k_heads = key_3d.shape[1]
pad_n_q_heads = triton.next_power_of_2(n_q_heads)
pad_n_k_heads = triton.next_power_of_2(n_k_heads)
pad_half_rotary_dim = triton.next_power_of_2(rotary_dim // 2)
_triton_bailing_mrope_forward[(num_tokens,)](
query_3d,
key_3d,
positions,
cos_sin_cache,
query_3d.stride(0),
query_3d.stride(1),
query_3d.stride(2),
key_3d.stride(0),
key_3d.stride(1),
key_3d.stride(2),
positions.stride(0),
positions.stride(1),
cos_sin_cache.stride(0),
cos_sin_cache.stride(1),
n_q_heads,
n_k_heads,
rotary_dim,
pad_n_q_heads,
pad_n_k_heads,
pad_half_rotary_dim,
spatial_size,
num_warps=4,
)
return query, key
class BailingMRotaryEmbedding(MRotaryEmbedding):
"""Bailing M-RoPE with alternating H/W frequencies and a temporal tail."""
def __init__(
self,
head_size: int,
rotary_dim: int,
max_position_embeddings: int,
base: float,
is_neox_style: bool,
dtype: torch.dtype,
mrope_section: list[int],
) -> None:
self.spatial_size = _validate_bailing_mrope_config(
head_size, rotary_dim, mrope_section
)
super().__init__(
head_size,
rotary_dim,
max_position_embeddings,
base,
is_neox_style,
dtype,
mrope_section=mrope_section,
)
def select_cos_sin(
self,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
_validate_bailing_positions(positions)
if cos_sin_cache is None:
cos_sin_cache = self.cos_sin_cache
cos_sin = cos_sin_cache[positions.to(torch.long)]
cos, sin = cos_sin.chunk(2, dim=-1)
if positions.ndim == 1:
return cos, sin
spatial_mask = torch.arange(self.spatial_size, device=cos.device) % 2 == 0
spatial_mask = spatial_mask.unsqueeze(0)
spatial_cos = torch.where(
spatial_mask,
cos[1, :, : self.spatial_size],
cos[2, :, : self.spatial_size],
)
spatial_sin = torch.where(
spatial_mask,
sin[1, :, : self.spatial_size],
sin[2, :, : self.spatial_size],
)
temporal_slice = slice(self.spatial_size, self.rotary_dim // 2)
return (
torch.cat((spatial_cos, cos[0, :, temporal_slice]), dim=-1),
torch.cat((spatial_sin, sin[0, :, temporal_slice]), dim=-1),
)
def forward_native(
self,
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor | None = None,
offsets: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
if key is None:
raise ValueError("Bailing M-RoPE requires key states")
cos_sin_cache = self._match_cos_sin_cache_dtype(query)
cos, sin = self.select_cos_sin(positions, cos_sin_cache)
num_tokens = positions.shape[-1]
query_shape = query.shape
query = query.view(num_tokens, -1, self.head_size)
query_rot = self.apply_rotary_emb.forward_native(
query[..., : self.rotary_dim],
cos,
sin,
)
query = torch.cat((query_rot, query[..., self.rotary_dim :]), dim=-1)
key_shape = key.shape
key = key.view(num_tokens, -1, self.head_size)
key_rot = self.apply_rotary_emb.forward_native(
key[..., : self.rotary_dim],
cos,
sin,
)
key = torch.cat((key_rot, key[..., self.rotary_dim :]), dim=-1)
return query.reshape(query_shape), key.reshape(key_shape)
def forward_cuda(
self,
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor | None = None,
offsets: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
if positions.ndim == 2 and not self.is_neox_style:
if key is None:
raise ValueError("Bailing M-RoPE requires key states")
cos_sin_cache = self._match_cos_sin_cache_dtype(query)
return triton_bailing_mrope(
query,
key,
positions,
cos_sin_cache,
self.spatial_size,
self.head_size,
self.rotary_dim,
)
return self.forward_native(positions, query, key, offsets)
def forward_cpu(
self,
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor | None = None,
offsets: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
return self.forward_native(positions, query, key, offsets)
+23 -4
View File
@@ -171,6 +171,27 @@ def _build_rope_parameters(config: PretrainedConfig) -> dict | None:
return rope_parameters or None
def _build_mla_rotary_embedding(
config: PretrainedConfig,
head_size: int,
) -> nn.Module:
rope_parameters = _build_rope_parameters(config)
if rope_parameters is not None and "mrope_section" in rope_parameters:
rope_type = rope_parameters.get("rope_type", "default")
if rope_type != "default":
raise ValueError(
f"Bailing M-RoPE only supports rope_type='default', got {rope_type!r}"
)
rope_parameters["rope_type"] = "bailing_mrope"
return get_rope(
head_size=head_size,
max_position=getattr(config, "max_position_embeddings", 8192),
is_neox_style=False,
rope_parameters=rope_parameters,
)
def _get_layer_swiglu_limit(limit_list: list | None, layer_idx: int) -> float | None:
if limit_list is None or layer_idx >= len(limit_list):
return None
@@ -524,11 +545,9 @@ class BailingMoeV3MLAAttention(nn.Module):
prefix=f"{prefix}.dense",
)
self.rotary_emb = get_rope(
self.rotary_emb = _build_mla_rotary_embedding(
config,
head_size=self.qk_rope_head_dim,
max_position=getattr(config, "max_position_embeddings", 8192),
is_neox_style=False,
rope_parameters=_build_rope_parameters(config),
)
mla_modules = MLAModules(
kv_a_layernorm=self.kv_a_layernorm,
@@ -33,9 +33,14 @@ from vllm.model_executor.models.bailing_moe_v3 import (
_maybe_pad_block_fp8_shared_expert_checkpoint_tensor,
_maybe_remap_ling_mxfp4_weight_names,
)
from vllm.model_executor.models.interfaces import SupportsPP
from vllm.model_executor.models.interfaces import (
MultiModalEmbeddings,
SupportsMultiModalEmbeddings,
SupportsPP,
)
from vllm.model_executor.models.utils import (
PPMissingLayer,
_merge_multimodal_embeddings,
is_pp_missing_parameter,
maybe_prefix,
)
@@ -214,8 +219,16 @@ class BailingMoeV3MultiTokenPredictor(nn.Module):
return self.logits_processor(head, mtp_layer.shared_head(hidden_states))
@support_torch_compile
class BailingMoeV3MTPModel(nn.Module, SupportsPP):
@support_torch_compile(
dynamic_arg_dims={
"input_ids": 0,
"positions": -1,
"hidden_states": 0,
"intermediate_tensors": 0,
"inputs_embeds": 0,
}
)
class BailingMoeV3MTPModel(nn.Module, SupportsPP, SupportsMultiModalEmbeddings):
hf_to_vllm_mapper = BailingMoeV3ForCausalLM.hf_to_vllm_mapper
packed_modules_mapping = {
@@ -242,8 +255,22 @@ class BailingMoeV3MTPModel(nn.Module, SupportsPP):
def share_lm_head(self, lm_head: nn.Module) -> None:
self.lm_head = lm_head
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def embed_input_ids(
self,
input_ids: torch.Tensor,
multimodal_embeddings: MultiModalEmbeddings | None = None,
*,
is_multimodal: torch.Tensor | None = None,
) -> torch.Tensor:
inputs_embeds = self.model.embed_input_ids(input_ids)
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
return inputs_embeds
assert is_multimodal is not None
return _merge_multimodal_embeddings(
inputs_embeds=inputs_embeds,
multimodal_embeddings=multimodal_embeddings,
is_multimodal=is_multimodal,
)
def forward(
self,
@@ -0,0 +1,527 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inference-only Bailing MoE V3 vision-language model."""
import copy
from collections.abc import Iterable, Mapping, Sequence
from typing import Any
import numpy as np
import regex as re
import torch
import torch.nn as nn
from transformers import AutoProcessor, BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
MultiModalFieldConfig,
MultiModalKwargsItems,
)
from vllm.multimodal.parse import MultiModalDataItems
from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
BaseMultiModalProcessor,
PromptReplacement,
PromptUpdate,
)
from vllm.multimodal.processing.processor import (
MultiModalPromptUpdates,
PlaceholderFeaturesInfo,
)
from vllm.sequence import IntermediateTensors
from .bailing_moe_v3 import BailingMoeV3ForCausalLM
from .interfaces import (
HasInnerState,
IsHybrid,
MultiModalEmbeddings,
SupportsMRoPE,
SupportsMultiModal,
SupportsPP,
_require_is_multimodal,
)
from .qwen2_5_vl import (
Qwen2_5_VLImageEmbeddingInputs,
Qwen2_5_VLImageInputs,
Qwen2_5_VLImagePixelInputs,
)
from .qwen2_vl import (
Qwen2VLProcessingInfo,
_create_qwen2vl_field_factory,
)
from .qwen3_vl import Qwen3_VisionTransformer
from .utils import (
AutoWeightsLoader,
WeightsMapper,
_merge_multimodal_embeddings,
maybe_prefix,
)
IMAGE_TOKEN = "<|image_pad|>"
IMAGE_PLACEHOLDER = f"<|vision_start|>{IMAGE_TOKEN}<|vision_end|>"
class BailingMoeV3VLProcessingInfo(Qwen2VLProcessingInfo):
"""Processing metadata for the remote Bailing multimodal processor."""
def get_hf_config(self):
return self.ctx.get_hf_config()
def get_hf_processor(self, **kwargs: object) -> AutoProcessor:
# The checkpoint supplies its processor through trust_remote_code.
return self.ctx.get_hf_processor(**kwargs)
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": None}
def get_mm_max_tokens_per_item(
self,
seq_len: int,
mm_counts: Mapping[str, int],
) -> Mapping[str, int]:
return {"image": self.get_max_image_tokens()}
class BailingMoeV3VLDummyInputsBuilder(
BaseDummyInputsBuilder[BailingMoeV3VLProcessingInfo]
):
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
return IMAGE_PLACEHOLDER * mm_counts.get("image", 0)
def get_dummy_mm_data(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
) -> MultiModalDataDict:
target_width, target_height = self.info.get_image_size_with_most_features()
return {
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
)
}
class BailingMoeV3VLMultiModalProcessor(
BaseMultiModalProcessor[BailingMoeV3VLProcessingInfo]
):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _apply_prompt_updates(
self,
token_ids: list[int],
mm_prompt_updates: MultiModalPromptUpdates,
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
if image_updates := mm_prompt_updates.get("image"):
tokenizer = self.info.get_tokenizer()
# HF adds a newline after each image before tokenization. Re-encode
# the text so it can merge with existing whitespace at the boundary.
text = re.sub(
f"{re.escape(IMAGE_PLACEHOLDER)}|{re.escape(IMAGE_TOKEN)}",
lambda _: IMAGE_PLACEHOLDER + "\n",
tokenizer.decode(token_ids),
count=len(image_updates),
)
token_ids = tokenizer.encode(text, add_special_tokens=False)
return super()._apply_prompt_updates(token_ids, mm_prompt_updates)
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
hf_processor_mm_kwargs: Mapping[str, object],
) -> Mapping[str, MultiModalFieldConfig]:
merge_size = self.info.get_hf_config().vision_config.spatial_merge_size
fields = _create_qwen2vl_field_factory(merge_size)(hf_inputs)
return {
key: fields[key]
for key in ("pixel_values", "image_embeds", "image_grid_thw")
}
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, Any],
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
tokenizer = self.info.get_tokenizer()
image_token_id = tokenizer.get_vocab()[IMAGE_TOKEN]
merge_size = self.info.get_hf_config().vision_config.spatial_merge_size
merge_unit = merge_size**2
def get_image_replacement(item_idx: int) -> list[int]:
out_item = out_mm_kwargs["image"][item_idx]
image_grid_thw = out_item["image_grid_thw"].data
assert isinstance(image_grid_thw, torch.Tensor)
num_image_tokens = int(image_grid_thw.prod()) // merge_unit
return [image_token_id] * num_image_tokens
return [
PromptReplacement(
modality="image",
target=[image_token_id],
replacement=get_image_replacement,
)
]
class BailingMoeV3VisionPatchMerger(nn.Module):
"""Normalize patches and merge each 2x2 group without an MLP."""
def __init__(
self,
context_dim: int,
spatial_merge_size: int,
norm_eps: float = 1e-6,
) -> None:
super().__init__()
self.hidden_size = context_dim * spatial_merge_size**2
self.norm = nn.LayerNorm(context_dim, eps=norm_eps)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
hidden_states = self.norm(hidden_states)
return hidden_states.reshape(-1, self.hidden_size)
class BailingMoeV3VisionTransformer(Qwen3_VisionTransformer):
"""Qwen3 ViT with Bailing's parameter-free patch merger."""
def __init__(
self,
vision_config: Any,
norm_eps: float = 1e-6,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
# Honor whole-tower exclusions before the parent creates linear layers.
if (
isinstance(quant_config, Fp8Config)
and prefix in quant_config.ignored_layers
):
quant_config = None
vision_config = copy.deepcopy(vision_config)
vision_config.deepstack_visual_indexes = []
merged_hidden_size = (
vision_config.hidden_size * vision_config.spatial_merge_size**2
)
vision_config.out_hidden_size = merged_hidden_size
super().__init__(
vision_config,
norm_eps=norm_eps,
quant_config=quant_config,
prefix=prefix,
)
self.merger = BailingMoeV3VisionPatchMerger(
context_dim=vision_config.hidden_size,
spatial_merge_size=vision_config.spatial_merge_size,
norm_eps=norm_eps,
)
self.deepstack_visual_indexes = []
self.deepstack_merger_list = nn.ModuleList()
self.out_hidden_size = merged_hidden_size
class BailingMoeV3VLProjector(nn.Module):
"""Tensor-parallel MLP mapping merged vision features to text width."""
def __init__(
self,
vision_hidden_size: int,
text_hidden_size: int,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
if (
isinstance(quant_config, Fp8Config)
and prefix in quant_config.ignored_layers
):
quant_config = None
self.linear_fc1 = ColumnParallelLinear(
vision_hidden_size,
text_hidden_size,
bias=True,
quant_config=quant_config,
prefix=maybe_prefix(prefix, "linear_fc1"),
)
self.act_fn = nn.GELU()
self.linear_fc2 = RowParallelLinear(
text_hidden_size,
text_hidden_size,
bias=True,
quant_config=quant_config,
prefix=maybe_prefix(prefix, "linear_fc2"),
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
hidden_states, _ = self.linear_fc1(hidden_states)
hidden_states = self.act_fn(hidden_states)
hidden_states, _ = self.linear_fc2(hidden_states)
return hidden_states
@MULTIMODAL_REGISTRY.register_processor(
BailingMoeV3VLMultiModalProcessor,
info=BailingMoeV3VLProcessingInfo,
dummy_inputs=BailingMoeV3VLDummyInputsBuilder,
)
class BailingMoeV3VLForConditionalGeneration(
nn.Module,
HasInnerState,
IsHybrid,
SupportsMultiModal,
SupportsPP,
SupportsMRoPE,
):
"""Native vLLM wrapper for ``BailingMoeV3VLConfig`` checkpoints."""
packed_modules_mapping = {
**BailingMoeV3ForCausalLM.packed_modules_mapping,
# The vision checkpoint already stores Q/K/V in one tensor.
"qkv": ["qkv"],
}
hf_to_vllm_mapper = BailingMoeV3ForCausalLM.hf_to_vllm_mapper | WeightsMapper(
orig_to_new_prefix={
# Match module names as well as parameter names for quantization.
"model.visual": "visual",
"model.linear_proj": "linear_proj",
"linear_proj.0": "linear_proj.linear_fc1",
"linear_proj.2": "linear_proj.linear_fc2",
"lm_head": "language_model.lm_head",
"model.": "language_model.model.",
}
)
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
return IMAGE_PLACEHOLDER
raise ValueError("Only image modality is supported")
@classmethod
def get_mamba_state_shape_from_config(
cls, vllm_config: VllmConfig
) -> tuple[tuple[int, int], tuple[int, int, int]]:
text_config = vllm_config.model_config.hf_config.text_config
return BailingMoeV3ForCausalLM.get_mamba_state_shape_from_config(
vllm_config.with_hf_config(text_config)
)
@classmethod
def get_mamba_state_dtype_from_config(
cls, vllm_config: VllmConfig
) -> tuple[torch.dtype, torch.dtype]:
text_config = vllm_config.model_config.hf_config.text_config
return BailingMoeV3ForCausalLM.get_mamba_state_dtype_from_config(
vllm_config.with_hf_config(text_config)
)
@classmethod
def get_mamba_state_copy_func(cls) -> tuple:
return BailingMoeV3ForCausalLM.get_mamba_state_copy_func()
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "model") -> None:
super().__init__()
config = vllm_config.model_config.hf_config
vision_config = config.vision_config
text_config = config.text_config
quant_config = vllm_config.quant_config
self.config = config
self.model_config = vllm_config.model_config
merged_vision_size = vision_config.hidden_size * (
vision_config.spatial_merge_size**2
)
with self._mark_tower_model(vllm_config, "image"):
self.visual = BailingMoeV3VisionTransformer(
vision_config,
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
quant_config=quant_config,
prefix=maybe_prefix(prefix, "visual"),
)
self.linear_proj = BailingMoeV3VLProjector(
vision_hidden_size=merged_vision_size,
text_hidden_size=text_config.hidden_size,
quant_config=quant_config,
prefix=maybe_prefix(prefix, "linear_proj"),
)
with self._mark_language_model(vllm_config):
self.language_model = BailingMoeV3ForCausalLM(
vllm_config=vllm_config.with_hf_config(text_config),
prefix=maybe_prefix(prefix, "language_model"),
)
self.make_empty_intermediate_tensors = (
self.language_model.make_empty_intermediate_tensors
)
def _parse_and_validate_image_input(
self, **kwargs: object
) -> Qwen2_5_VLImageInputs | None:
pixel_values = kwargs.pop("pixel_values", None)
image_embeds = kwargs.pop("image_embeds", None)
image_grid_thw = kwargs.pop("image_grid_thw", None)
if pixel_values is None and image_embeds is None:
return None
if pixel_values is not None:
return Qwen2_5_VLImagePixelInputs(
type="pixel_values",
pixel_values=pixel_values,
image_grid_thw=image_grid_thw,
)
return Qwen2_5_VLImageEmbeddingInputs(
type="image_embeds",
image_embeds=image_embeds,
image_grid_thw=image_grid_thw,
)
def _process_image_input(
self, image_input: Qwen2_5_VLImageInputs
) -> tuple[torch.Tensor, ...]:
image_grid_thw = image_input["image_grid_thw"]
assert image_grid_thw.ndim == 2
if image_input["type"] == "image_embeds":
image_embeds = image_input["image_embeds"]
else:
pixel_values = image_input["pixel_values"].type(self.visual.dtype)
vision_features = self.visual(
pixel_values,
grid_thw=image_grid_thw,
)
image_embeds = self.linear_proj(vision_features)
image_embeds = image_embeds.to(dtype=self.visual.dtype)
merge_unit = self.visual.spatial_merge_size**2
sizes = (image_grid_thw.prod(-1) // merge_unit).tolist()
return image_embeds.split(sizes)
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
image_input = self._parse_and_validate_image_input(**kwargs)
if image_input is None:
return []
return self._process_image_input(image_input)
def embed_input_ids(
self,
input_ids: torch.Tensor,
multimodal_embeddings: MultiModalEmbeddings | None = None,
*,
is_multimodal: torch.Tensor | None = None,
) -> torch.Tensor:
inputs_embeds = self._embed_text_input_ids(
input_ids,
self.language_model.embed_input_ids,
is_multimodal=is_multimodal,
)
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
return inputs_embeds
return _merge_multimodal_embeddings(
inputs_embeds=inputs_embeds,
multimodal_embeddings=multimodal_embeddings,
is_multimodal=_require_is_multimodal(is_multimodal),
)
def get_mrope_input_positions(
self,
input_tokens: list[int],
mm_features: list[MultiModalFeatureSpec],
) -> tuple[torch.Tensor, int]:
position_blocks: list[np.ndarray] = []
consumed = 0
merge_size = self.config.vision_config.spatial_merge_size
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
if mm_feature.modality != "image":
raise ValueError(f"Unsupported modality: {mm_feature.modality}")
offset = mm_feature.mm_position.offset
text_len = offset - consumed
start = position_blocks[-1].max() + 1 if position_blocks else 0
position_blocks.append(
np.broadcast_to(np.arange(text_len), (3, text_len)) + start
)
assert mm_feature.data is not None
image_grid_thw = mm_feature.data["image_grid_thw"].data
assert isinstance(image_grid_thw, torch.Tensor)
t, h, w = image_grid_thw.tolist()
assert t == 1, f"Image must have one temporal grid, got {t}"
grid = np.indices((t, h // merge_size, w // merge_size))
position_blocks.append(grid.reshape(3, -1) + text_len + start)
consumed = offset + t * (h // merge_size) * (w // merge_size)
if consumed < len(input_tokens):
start = position_blocks[-1].max() + 1 if position_blocks else 0
text_len = len(input_tokens) - consumed
position_blocks.append(
np.broadcast_to(np.arange(text_len), (3, text_len)) + start
)
if not position_blocks:
positions = np.empty((3, 0), dtype=np.int64)
return torch.from_numpy(positions), 0
positions = np.concatenate(position_blocks, axis=1).reshape(3, -1)
position_delta = int(positions.max() + 1 - len(input_tokens))
return torch.from_numpy(positions), position_delta
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> torch.Tensor | IntermediateTensors:
if intermediate_tensors is not None:
inputs_embeds = None
return self.language_model(
input_ids=input_ids,
positions=positions,
intermediate_tensors=intermediate_tensors,
inputs_embeds=inputs_embeds,
)
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
return self.language_model.compute_logits(hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def get_mm_mapping(self) -> MultiModelKeys:
return MultiModelKeys.from_string_field(
language_model="language_model",
connector="linear_proj",
tower_model="visual.",
)
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
return num_image_tokens * self.visual.spatial_merge_size**2
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
return num_vision_tokens // self.visual.spatial_merge_size**2
+4
View File
@@ -350,6 +350,10 @@ _MULTIMODAL_MODELS = {
"AudioFlamingo3ForConditionalGeneration",
),
"BagelForConditionalGeneration": ("bagel", "BagelForConditionalGeneration"),
"BailingMoeV3VLForConditionalGeneration": (
"bailing_moe_v3_vl",
"BailingMoeV3VLForConditionalGeneration",
),
"BeeForConditionalGeneration": ("bee", "BeeForConditionalGeneration"),
"Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
"Cohere2VisionForConditionalGeneration": (