[Model] Add Bailing V3 VL support (#55921)

Signed-off-by: zexplorerhj <zhjoneson@163.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
zexplorerhj
2026-09-10 02:46:56 -07:00
committed by GitHub
co-authored by Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 6ff479e1f7
commit fe26c705f7
19 changed files with 1825 additions and 20 deletions
+23 -4
View File
@@ -171,6 +171,27 @@ def _build_rope_parameters(config: PretrainedConfig) -> dict | None:
return rope_parameters or None
def _build_mla_rotary_embedding(
config: PretrainedConfig,
head_size: int,
) -> nn.Module:
rope_parameters = _build_rope_parameters(config)
if rope_parameters is not None and "mrope_section" in rope_parameters:
rope_type = rope_parameters.get("rope_type", "default")
if rope_type != "default":
raise ValueError(
f"Bailing M-RoPE only supports rope_type='default', got {rope_type!r}"
)
rope_parameters["rope_type"] = "bailing_mrope"
return get_rope(
head_size=head_size,
max_position=getattr(config, "max_position_embeddings", 8192),
is_neox_style=False,
rope_parameters=rope_parameters,
)
def _get_layer_swiglu_limit(limit_list: list | None, layer_idx: int) -> float | None:
if limit_list is None or layer_idx >= len(limit_list):
return None
@@ -524,11 +545,9 @@ class BailingMoeV3MLAAttention(nn.Module):
prefix=f"{prefix}.dense",
)
self.rotary_emb = get_rope(
self.rotary_emb = _build_mla_rotary_embedding(
config,
head_size=self.qk_rope_head_dim,
max_position=getattr(config, "max_position_embeddings", 8192),
is_neox_style=False,
rope_parameters=_build_rope_parameters(config),
)
mla_modules = MLAModules(
kv_a_layernorm=self.kv_a_layernorm,