[Model] Add Bailing V3 VL support (#55921)

Signed-off-by: zexplorerhj <zhjoneson@163.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
zexplorerhj
2026-09-10 02:46:56 -07:00
committed by GitHub
co-authored by Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 6ff479e1f7
commit fe26c705f7
19 changed files with 1825 additions and 20 deletions
@@ -0,0 +1,126 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Bailing prompt expansion tests without downloading a checkpoint."""
from types import SimpleNamespace
import pytest
import torch
from PIL import Image
from tokenizers import Tokenizer, decoders, models, pre_tokenizers
from transformers import PreTrainedTokenizerFast
from vllm.model_executor.models.bailing_moe_v3_vl import (
IMAGE_PLACEHOLDER,
IMAGE_TOKEN,
BailingMoeV3VLMultiModalProcessor,
)
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataItems
pytestmark = pytest.mark.cpu_test
@pytest.fixture
def processor():
vocab = {
token: index
for index, token in enumerate(sorted(pre_tokenizers.ByteLevel.alphabet()))
}
# A newline added by the processor must merge with an existing newline.
vocab["ĊĊ"] = len(vocab)
backend = Tokenizer(models.BPE(vocab=vocab, merges=[("Ċ", "Ċ")]))
backend.pre_tokenizer = pre_tokenizers.ByteLevel(
add_prefix_space=False, use_regex=False
)
backend.decoder = decoders.ByteLevel()
tokenizer = PreTrainedTokenizerFast(
tokenizer_object=backend,
additional_special_tokens=[
"<|vision_start|>",
IMAGE_TOKEN,
"<|vision_end|>",
],
clean_up_tokenization_spaces=False,
)
processor = object.__new__(BailingMoeV3VLMultiModalProcessor)
processor.info = SimpleNamespace(
get_tokenizer=lambda: tokenizer,
get_hf_config=lambda: SimpleNamespace(
vision_config=SimpleNamespace(spatial_merge_size=2)
),
)
return processor
@pytest.mark.parametrize(
("prompt", "expected_text", "token_counts"),
[
(
f"before{IMAGE_PLACEHOLDER}after",
f"before<|vision_start|>{IMAGE_TOKEN * 2}<|vision_end|>\nafter",
[2],
),
(
f"before{IMAGE_TOKEN}after",
f"before<|vision_start|>{IMAGE_TOKEN * 2}<|vision_end|>\nafter",
[2],
),
(
f"before{IMAGE_PLACEHOLDER}\nafter",
f"before<|vision_start|>{IMAGE_TOKEN * 2}<|vision_end|>\n\nafter",
[2],
),
(
f"before{IMAGE_PLACEHOLDER}{IMAGE_PLACEHOLDER}after",
f"before<|vision_start|>{IMAGE_TOKEN * 2}<|vision_end|>\n"
f"<|vision_start|>{IMAGE_TOKEN * 3}<|vision_end|>\nafter",
[2, 3],
),
],
ids=["wrapped", "bare", "existing_newline", "adjacent_images"],
)
def test_image_expansion_matches_wrappers_and_newline_tokenization(
processor, prompt, expected_text, token_counts
):
tokenizer = processor.info.get_tokenizer()
image_id = tokenizer.get_vocab()[IMAGE_TOKEN]
grids = torch.tensor([[1, 2, 2 * count] for count in token_counts])
mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
{"image_grid_thw": grids},
{"image_grid_thw": MultiModalFieldConfig.batched("image")},
)
mm_items = MultiModalDataItems(
{"image": ImageProcessorItems([Image.new("RGB", (1, 1))] * len(token_counts))}
)
updates = processor._get_mm_prompt_updates(mm_items, {}, mm_kwargs)
prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
expected_ids = tokenizer.encode(expected_text, add_special_tokens=False)
actual_ids, placeholders = processor._apply_prompt_updates(prompt_ids, updates)
assert actual_ids == expected_ids
assert len(placeholders["image"]) == len(token_counts)
found = processor._find_mm_placeholders(actual_ids, updates)
search_offset = 0
for index, count in enumerate(token_counts):
placeholder = placeholders["image"][index]
expected_offset = expected_ids.index(image_id, search_offset)
assert placeholder.start_idx == expected_offset
assert placeholder.tokens == [image_id] * count
assert placeholder.to_range().get_num_embeds() == count
assert placeholder.to_range() == found["image"][index].to_range()
search_offset = expected_offset + count
def test_text_only_prompt_preserves_original_token_ids(processor):
tokenizer = processor.info.get_tokenizer()
prompt_ids = tokenizer.encode("before\n", add_special_tokens=False)
prompt_ids += tokenizer.encode("\nafter", add_special_tokens=False)
# Deliberately use two newline tokens instead of their merged BPE token.
assert prompt_ids != tokenizer.encode("before\n\nafter", add_special_tokens=False)
actual_ids, placeholders = processor._apply_prompt_updates(prompt_ids, {})
assert actual_ids == prompt_ids
assert placeholders == {}
+30
View File
@@ -184,6 +184,36 @@ def test_cosmos3_edge_checkpoint_weights_mapper():
)
@pytest.mark.cpu_test
def test_bailing_vl_mapper_handles_module_and_parameter_names():
from vllm.model_executor.models.bailing_moe_v3_vl import (
BailingMoeV3VLForConditionalGeneration,
)
mapper = BailingMoeV3VLForConditionalGeneration.hf_to_vllm_mapper
names = {
"model.visual": "visual",
"model.visual.blocks.0.attn.qkv.weight": "visual.blocks.0.attn.qkv.weight",
"model.linear_proj": "linear_proj",
"linear_proj.0": "linear_proj.linear_fc1",
"model.linear_proj.0.weight": "linear_proj.linear_fc1.weight",
"linear_proj.2": "linear_proj.linear_fc2",
"linear_proj.2.weight_scale": "linear_proj.linear_fc2.weight_scale",
"model.linear_proj.2.bias": "linear_proj.linear_fc2.bias",
"linear_proj.linear_fc1.weight": "linear_proj.linear_fc1.weight",
"model.linear_proj.linear_fc2.weight": "linear_proj.linear_fc2.weight",
"model.layers.0.attention.q_proj.weight": (
"language_model.model.layers.0.self_attn.q_proj.weight"
),
"model.layers.0.mlp.experts.3.up_proj.weight": (
"language_model.model.layers.0.mlp.experts.3.up_proj.weight"
),
"lm_head": "language_model.lm_head",
"lm_head.weight": "language_model.lm_head.weight",
}
assert mapper.apply_list(list(names)) == list(names.values())
def create_repo_dummy_weights(repo: str) -> Iterable[tuple[str, torch.Tensor]]:
"""Create weights from safetensors checkpoint metadata"""
metadata = try_get_safetensors_metadata(repo)
+12
View File
@@ -796,6 +796,18 @@ _MULTIMODAL_EXAMPLE_MODELS = {
},
),
"BagelForConditionalGeneration": _HfExamplesInfo("ByteDance-Seed/BAGEL-7B-MoT"),
"BailingMoeV3VLForConditionalGeneration": _HfExamplesInfo(
"inclusionAI/Ling-3.0-flash-VL",
trust_remote_code=True,
is_available_online=True,
use_original_num_layers=True,
hf_overrides={
"text_config": {
"num_hidden_layers": 6,
"layer_types": ["linear_attention"] * 5 + ["full_attention"],
}
},
),
"BeeForConditionalGeneration": _HfExamplesInfo(
"Open-Bee/Bee-8B-RL",
trust_remote_code=True,