[Model] Add Bailing V3 VL support (#55921)

Signed-off-by: zexplorerhj <zhjoneson@163.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
zexplorerhj
2026-09-10 02:46:56 -07:00
committed by GitHub
co-authored by Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 6ff479e1f7
commit fe26c705f7
19 changed files with 1825 additions and 20 deletions
+176
View File
@@ -0,0 +1,176 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
from types import SimpleNamespace
import pytest
import torch
from vllm.config import DeviceConfig, VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.rotary_embedding.bailing_mrope import (
BailingMRotaryEmbedding,
)
from vllm.model_executor.models.bailing_moe_v3 import _build_mla_rotary_embedding
from vllm.platforms import current_platform
def test_bailing_mrope_selects_alternating_spatial_then_temporal():
with set_current_vllm_config(VllmConfig(device_config=DeviceConfig("cpu"))):
rotary_emb = BailingMRotaryEmbedding(
head_size=12,
rotary_dim=12,
max_position_embeddings=8,
base=10_000,
is_neox_style=False,
dtype=torch.float32,
mrope_section=[2, 2, 2],
)
positions = torch.tensor([[1, 2], [3, 4], [5, 6]])
frequencies = torch.arange(6).expand(8, -1)
position_values = torch.arange(8).unsqueeze(1) * 10
cos = position_values + frequencies
sin = cos + 100
cos_sin_cache = torch.cat((cos, sin), dim=-1)
actual_cos, actual_sin = rotary_emb.select_cos_sin(
positions,
cos_sin_cache,
)
expected_cos = torch.tensor(
[
[30, 51, 32, 53, 14, 15],
[40, 61, 42, 63, 24, 25],
]
)
torch.testing.assert_close(actual_cos, expected_cos)
torch.testing.assert_close(actual_sin, expected_cos + 100)
def test_bailing_factory_preserves_mla_width_and_config():
config = SimpleNamespace(
max_position_embeddings=128,
rope_parameters={
"rope_type": "default",
"rope_theta": 6_000_000,
"partial_rotary_factor": 0.5,
"mrope_section": [8, 12, 12],
},
rope_scaling={"partial_rotary_factor": 0.5},
)
original = copy.deepcopy(vars(config))
with set_current_vllm_config(VllmConfig(device_config=DeviceConfig("cpu"))):
rope = _build_mla_rotary_embedding(config, head_size=64)
assert rope is _build_mla_rotary_embedding(config, head_size=64)
generic = get_rope(
head_size=64,
max_position=128,
is_neox_style=False,
rope_parameters={
"rope_type": "default",
"rope_theta": 6_000_000,
"mrope_section": [8, 12, 12],
},
)
assert isinstance(rope, BailingMRotaryEmbedding)
assert rope is not generic
assert rope.rotary_dim == 64
assert vars(config) == original
def test_bailing_factory_rejects_unsupported_scaling():
config = SimpleNamespace(
rope_parameters={
"rope_type": "yarn",
"factor": 2.0,
"mrope_section": [8, 12, 12],
}
)
with pytest.raises(ValueError, match="only supports rope_type='default'"):
_build_mla_rotary_embedding(config, head_size=64)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="Skipping CUDA/ROCm-only test.",
)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("position_kind", ["text", "multimodal"])
@torch.inference_mode()
def test_bailing_mrope_cuda_matches_native_for_strided_mla_views(
dtype: torch.dtype,
position_kind: str,
):
device = torch.device("cuda")
num_tokens = 17
num_query_heads = 8
with set_current_vllm_config(VllmConfig()):
rotary_emb = BailingMRotaryEmbedding(
head_size=64,
rotary_dim=64,
max_position_embeddings=64,
base=6_000_000,
is_neox_style=False,
dtype=dtype,
mrope_section=[8, 12, 12],
).to(device)
torch.manual_seed(0)
multimodal_positions = torch.randint(
0,
64,
(3, 2 * num_tokens),
dtype=torch.int32,
device=device,
)[:, ::2]
if position_kind == "multimodal":
positions = multimodal_positions
reference_positions = positions
else:
positions = torch.randint(
0,
64,
(2 * num_tokens,),
dtype=torch.int64,
device=device,
)[::2]
reference_positions = positions.expand(3, -1)
query_buffer = torch.randn(
num_tokens,
num_query_heads,
192,
dtype=dtype,
device=device,
)
key_buffer = torch.randn(
num_tokens,
576,
dtype=dtype,
device=device,
)
query = query_buffer[..., 128:]
key = key_buffer[:, 512:].unsqueeze(1)
assert not query.is_contiguous()
assert not key.is_contiguous()
query_prefix = query_buffer[..., :128].clone()
key_prefix = key_buffer[:, :512].clone()
expected_query, expected_key = rotary_emb.forward_native(
reference_positions,
query.clone(),
key.clone(),
)
actual_query, actual_key = rotary_emb.forward_cuda(positions, query, key)
assert expected_key is not None
assert actual_key is not None
torch.testing.assert_close(actual_query, expected_query, atol=1e-2, rtol=1.6e-2)
torch.testing.assert_close(actual_key, expected_key, atol=1e-2, rtol=1.6e-2)
assert torch.equal(query_buffer[..., :128], query_prefix)
assert torch.equal(key_buffer[:, :512], key_prefix)
if position_kind == "multimodal":
assert actual_query.data_ptr() == query.data_ptr()
assert actual_key.data_ptr() == key.data_ptr()
@@ -0,0 +1,126 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Bailing prompt expansion tests without downloading a checkpoint."""
from types import SimpleNamespace
import pytest
import torch
from PIL import Image
from tokenizers import Tokenizer, decoders, models, pre_tokenizers
from transformers import PreTrainedTokenizerFast
from vllm.model_executor.models.bailing_moe_v3_vl import (
IMAGE_PLACEHOLDER,
IMAGE_TOKEN,
BailingMoeV3VLMultiModalProcessor,
)
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataItems
pytestmark = pytest.mark.cpu_test
@pytest.fixture
def processor():
vocab = {
token: index
for index, token in enumerate(sorted(pre_tokenizers.ByteLevel.alphabet()))
}
# A newline added by the processor must merge with an existing newline.
vocab["ĊĊ"] = len(vocab)
backend = Tokenizer(models.BPE(vocab=vocab, merges=[("Ċ", "Ċ")]))
backend.pre_tokenizer = pre_tokenizers.ByteLevel(
add_prefix_space=False, use_regex=False
)
backend.decoder = decoders.ByteLevel()
tokenizer = PreTrainedTokenizerFast(
tokenizer_object=backend,
additional_special_tokens=[
"<|vision_start|>",
IMAGE_TOKEN,
"<|vision_end|>",
],
clean_up_tokenization_spaces=False,
)
processor = object.__new__(BailingMoeV3VLMultiModalProcessor)
processor.info = SimpleNamespace(
get_tokenizer=lambda: tokenizer,
get_hf_config=lambda: SimpleNamespace(
vision_config=SimpleNamespace(spatial_merge_size=2)
),
)
return processor
@pytest.mark.parametrize(
("prompt", "expected_text", "token_counts"),
[
(
f"before{IMAGE_PLACEHOLDER}after",
f"before<|vision_start|>{IMAGE_TOKEN * 2}<|vision_end|>\nafter",
[2],
),
(
f"before{IMAGE_TOKEN}after",
f"before<|vision_start|>{IMAGE_TOKEN * 2}<|vision_end|>\nafter",
[2],
),
(
f"before{IMAGE_PLACEHOLDER}\nafter",
f"before<|vision_start|>{IMAGE_TOKEN * 2}<|vision_end|>\n\nafter",
[2],
),
(
f"before{IMAGE_PLACEHOLDER}{IMAGE_PLACEHOLDER}after",
f"before<|vision_start|>{IMAGE_TOKEN * 2}<|vision_end|>\n"
f"<|vision_start|>{IMAGE_TOKEN * 3}<|vision_end|>\nafter",
[2, 3],
),
],
ids=["wrapped", "bare", "existing_newline", "adjacent_images"],
)
def test_image_expansion_matches_wrappers_and_newline_tokenization(
processor, prompt, expected_text, token_counts
):
tokenizer = processor.info.get_tokenizer()
image_id = tokenizer.get_vocab()[IMAGE_TOKEN]
grids = torch.tensor([[1, 2, 2 * count] for count in token_counts])
mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
{"image_grid_thw": grids},
{"image_grid_thw": MultiModalFieldConfig.batched("image")},
)
mm_items = MultiModalDataItems(
{"image": ImageProcessorItems([Image.new("RGB", (1, 1))] * len(token_counts))}
)
updates = processor._get_mm_prompt_updates(mm_items, {}, mm_kwargs)
prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
expected_ids = tokenizer.encode(expected_text, add_special_tokens=False)
actual_ids, placeholders = processor._apply_prompt_updates(prompt_ids, updates)
assert actual_ids == expected_ids
assert len(placeholders["image"]) == len(token_counts)
found = processor._find_mm_placeholders(actual_ids, updates)
search_offset = 0
for index, count in enumerate(token_counts):
placeholder = placeholders["image"][index]
expected_offset = expected_ids.index(image_id, search_offset)
assert placeholder.start_idx == expected_offset
assert placeholder.tokens == [image_id] * count
assert placeholder.to_range().get_num_embeds() == count
assert placeholder.to_range() == found["image"][index].to_range()
search_offset = expected_offset + count
def test_text_only_prompt_preserves_original_token_ids(processor):
tokenizer = processor.info.get_tokenizer()
prompt_ids = tokenizer.encode("before\n", add_special_tokens=False)
prompt_ids += tokenizer.encode("\nafter", add_special_tokens=False)
# Deliberately use two newline tokens instead of their merged BPE token.
assert prompt_ids != tokenizer.encode("before\n\nafter", add_special_tokens=False)
actual_ids, placeholders = processor._apply_prompt_updates(prompt_ids, {})
assert actual_ids == prompt_ids
assert placeholders == {}
+30
View File
@@ -184,6 +184,36 @@ def test_cosmos3_edge_checkpoint_weights_mapper():
)
@pytest.mark.cpu_test
def test_bailing_vl_mapper_handles_module_and_parameter_names():
from vllm.model_executor.models.bailing_moe_v3_vl import (
BailingMoeV3VLForConditionalGeneration,
)
mapper = BailingMoeV3VLForConditionalGeneration.hf_to_vllm_mapper
names = {
"model.visual": "visual",
"model.visual.blocks.0.attn.qkv.weight": "visual.blocks.0.attn.qkv.weight",
"model.linear_proj": "linear_proj",
"linear_proj.0": "linear_proj.linear_fc1",
"model.linear_proj.0.weight": "linear_proj.linear_fc1.weight",
"linear_proj.2": "linear_proj.linear_fc2",
"linear_proj.2.weight_scale": "linear_proj.linear_fc2.weight_scale",
"model.linear_proj.2.bias": "linear_proj.linear_fc2.bias",
"linear_proj.linear_fc1.weight": "linear_proj.linear_fc1.weight",
"model.linear_proj.linear_fc2.weight": "linear_proj.linear_fc2.weight",
"model.layers.0.attention.q_proj.weight": (
"language_model.model.layers.0.self_attn.q_proj.weight"
),
"model.layers.0.mlp.experts.3.up_proj.weight": (
"language_model.model.layers.0.mlp.experts.3.up_proj.weight"
),
"lm_head": "language_model.lm_head",
"lm_head.weight": "language_model.lm_head.weight",
}
assert mapper.apply_list(list(names)) == list(names.values())
def create_repo_dummy_weights(repo: str) -> Iterable[tuple[str, torch.Tensor]]:
"""Create weights from safetensors checkpoint metadata"""
metadata = try_get_safetensors_metadata(repo)
+12
View File
@@ -796,6 +796,18 @@ _MULTIMODAL_EXAMPLE_MODELS = {
},
),
"BagelForConditionalGeneration": _HfExamplesInfo("ByteDance-Seed/BAGEL-7B-MoT"),
"BailingMoeV3VLForConditionalGeneration": _HfExamplesInfo(
"inclusionAI/Ling-3.0-flash-VL",
trust_remote_code=True,
is_available_online=True,
use_original_num_layers=True,
hf_overrides={
"text_config": {
"num_hidden_layers": 6,
"layer_types": ["linear_attention"] * 5 + ["full_attention"],
}
},
),
"BeeForConditionalGeneration": _HfExamplesInfo(
"Open-Bee/Bee-8B-RL",
trust_remote_code=True,
+7 -6
View File
@@ -3,6 +3,8 @@
import json
import pytest
from tests.parser.engine.replay_harness import MockTokenizer, _test_request
from vllm.reasoning import ReasoningParserManager
from vllm.tool_parsers import ToolParserManager
@@ -57,19 +59,18 @@ def test_ling3_disable_thinking_keeps_reasoning_as_content():
assert content == "<think>reason</think>answer"
def test_ling3_enable_thinking_keeps_open_reasoning_as_content():
@pytest.mark.parametrize("model_output", ["only reasoning", "<think>only reasoning"])
def test_ling3_truncated_reasoning_stays_reasoning(model_output):
parser_cls = ReasoningParserManager.get_reasoning_parser("ling3")
parser = parser_cls(
_tokenizer(),
chat_template_kwargs={"enable_thinking": True},
)
reasoning, content = parser.extract_reasoning(
"<think>only reasoning", _test_request()
)
reasoning, content = parser.extract_reasoning(model_output, _test_request())
assert reasoning is None
assert content == "only reasoning"
assert reasoning == "only reasoning"
assert content is None
def test_ling3_tool_call_without_newline():
@@ -0,0 +1,159 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
import json
from pathlib import Path
from typing import Any
import pytest
from transformers import AutoConfig
from vllm.transformers_utils.config import get_config, uses_mrope
from vllm.transformers_utils.configs.bailing_moe_v3_vl import (
BailingMoeV3TextConfig,
BailingMoeV3VisionConfig,
BailingMoeV3VLConfig,
)
def _bailing_v3_vl_config() -> dict[str, Any]:
return {
"architectures": ["BailingMoeV3VLForConditionalGeneration"],
"auto_map": {
"AutoConfig": ("configuration_bailing_moe_v3_vl.BailingMoeV3VLConfig")
},
"model_type": "bailing_moe_v3_vl",
"mrope_section": [8, 12, 12],
"text_config": {
"num_hidden_layers": 42,
"layer_group_size": 6,
"rope_theta": 6_000_000,
},
"vision_config": {"disable_merger_proj": True},
}
def _write_config(path: Path, config: dict[str, Any]) -> None:
path.mkdir()
(path / "config.json").write_text(json.dumps(config), encoding="utf-8")
def test_bailing_v3_vl_config_loads_without_remote_code(tmp_path: Path):
model_path = tmp_path / "model"
_write_config(model_path, _bailing_v3_vl_config())
config = get_config(model_path, trust_remote_code=False)
assert isinstance(config, BailingMoeV3VLConfig)
assert isinstance(config.text_config, BailingMoeV3TextConfig)
assert isinstance(config.vision_config, BailingMoeV3VisionConfig)
assert isinstance(
AutoConfig.from_pretrained(model_path, trust_remote_code=False),
BailingMoeV3VLConfig,
)
assert config.mrope_section == [8, 12, 12]
assert uses_mrope(config)
text_config = config.text_config
assert text_config.model_type == "bailing_hybrid"
assert text_config.architectures == ["BailingMoeV3ForCausalLM"]
assert text_config.rope_parameters["mrope_section"] == [8, 12, 12]
assert text_config.rope_parameters["rope_theta"] == 6_000_000
assert text_config.layer_types.count("linear_attention") == 35
assert [
layer_idx
for layer_idx, layer_type in enumerate(text_config.layer_types)
if layer_type == "full_attention"
] == [5, 11, 17, 23, 29, 35, 41]
assert config.vision_config.disable_merger_proj is True
normalized = config.to_dict()
restored = BailingMoeV3VLConfig.from_dict(normalized)
assert isinstance(restored.text_config, BailingMoeV3TextConfig)
assert isinstance(restored.vision_config, BailingMoeV3VisionConfig)
assert restored.to_dict() == normalized
def test_bailing_v3_vl_config_preserves_explicit_nested_values():
config_dict = _bailing_v3_vl_config()
config_dict["architectures"] = ["CustomBailingVLForConditionalGeneration"]
text_config = config_dict["text_config"]
text_config["architectures"] = ["CustomBailingForCausalLM"]
text_config["layer_types"] = (["linear_attention"] * 5 + ["full_attention"]) * 7
text_config["rope_theta"] = 1_000_000
text_config["rope_parameters"] = {
"rope_type": "default",
"rope_theta": 1_000_000,
"mrope_section": [8, 12, 12],
}
original = copy.deepcopy(config_dict)
config = BailingMoeV3VLConfig(**config_dict)
assert config_dict == original
assert config.architectures == ["CustomBailingVLForConditionalGeneration"]
assert config.text_config.architectures == ["CustomBailingForCausalLM"]
assert config.text_config.layer_types == text_config["layer_types"]
assert config.text_config.rope_parameters["rope_theta"] == 1_000_000
assert config.text_config.rope_parameters["mrope_section"] == [8, 12, 12]
def test_bailing_v3_vl_config_normalizes_config_instance():
text_config = BailingMoeV3TextConfig()
config = BailingMoeV3VLConfig(
text_config=text_config,
mrope_section=[8, 12, 12],
)
assert config.text_config is text_config
assert text_config.rope_parameters["mrope_section"] == [8, 12, 12]
def test_bailing_v3_text_config_trailing_and_legacy_layers():
expected = ["linear_attention", "linear_attention", "full_attention"] * 2 + [
"full_attention"
] * 2
text_config = BailingMoeV3TextConfig(
num_hidden_layers=8,
layer_group_size=3,
)
assert text_config.layer_types == expected
text_config = BailingMoeV3TextConfig(
num_hidden_layers=8,
layer_group_size=3,
layers_block_type=["mamba", "mamba", "attention"] * 2 + ["attention"] * 2,
)
assert text_config.layer_types == expected
def test_bailing_v3_text_config_accepts_legacy_rope_type():
text_config = BailingMoeV3TextConfig(rope_scaling={"type": "linear", "factor": 2.0})
assert text_config.rope_parameters["type"] == "linear"
assert text_config.rope_parameters["rope_type"] == "linear"
def test_bailing_v3_vl_config_rejects_invalid_values():
with pytest.raises(ValueError, match="layer_group_size must be positive"):
BailingMoeV3TextConfig(layer_group_size=0)
with pytest.raises(ValueError, match="must match the layer_group_size schedule"):
BailingMoeV3TextConfig(
num_hidden_layers=8,
layer_group_size=3,
layer_types=["full_attention"] * 8,
)
with pytest.raises(ValueError, match="conflicting M-RoPE sections"):
BailingMoeV3VLConfig(
text_config={
"rope_parameters": {
"rope_type": "default",
"mrope_section": [4, 14, 14],
}
},
mrope_section=[8, 12, 12],
)