[Model] Add Bailing V3 VL support (#55921)
Signed-off-by: zexplorerhj <zhjoneson@163.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
6ff479e1f7
commit
fe26c705f7
@@ -171,6 +171,27 @@ def _build_rope_parameters(config: PretrainedConfig) -> dict | None:
|
||||
return rope_parameters or None
|
||||
|
||||
|
||||
def _build_mla_rotary_embedding(
|
||||
config: PretrainedConfig,
|
||||
head_size: int,
|
||||
) -> nn.Module:
|
||||
rope_parameters = _build_rope_parameters(config)
|
||||
if rope_parameters is not None and "mrope_section" in rope_parameters:
|
||||
rope_type = rope_parameters.get("rope_type", "default")
|
||||
if rope_type != "default":
|
||||
raise ValueError(
|
||||
f"Bailing M-RoPE only supports rope_type='default', got {rope_type!r}"
|
||||
)
|
||||
rope_parameters["rope_type"] = "bailing_mrope"
|
||||
|
||||
return get_rope(
|
||||
head_size=head_size,
|
||||
max_position=getattr(config, "max_position_embeddings", 8192),
|
||||
is_neox_style=False,
|
||||
rope_parameters=rope_parameters,
|
||||
)
|
||||
|
||||
|
||||
def _get_layer_swiglu_limit(limit_list: list | None, layer_idx: int) -> float | None:
|
||||
if limit_list is None or layer_idx >= len(limit_list):
|
||||
return None
|
||||
@@ -524,11 +545,9 @@ class BailingMoeV3MLAAttention(nn.Module):
|
||||
prefix=f"{prefix}.dense",
|
||||
)
|
||||
|
||||
self.rotary_emb = get_rope(
|
||||
self.rotary_emb = _build_mla_rotary_embedding(
|
||||
config,
|
||||
head_size=self.qk_rope_head_dim,
|
||||
max_position=getattr(config, "max_position_embeddings", 8192),
|
||||
is_neox_style=False,
|
||||
rope_parameters=_build_rope_parameters(config),
|
||||
)
|
||||
mla_modules = MLAModules(
|
||||
kv_a_layernorm=self.kv_a_layernorm,
|
||||
|
||||
Reference in New Issue
Block a user