[Bugfix][Model] K-EXAONE: restore loading of pre-2.0 configs and apply RoPE to the MTP layer (#60187)

Signed-off-by: godmook <cmoh4135@naver.com>
Signed-off-by: Mook <68294499+Godmook@users.noreply.github.com>
This commit is contained in:
Mook
2026-10-10 15:05:55 +00:00
committed by GitHub
parent 2bea22002d
commit bf16c9c139
3 changed files with 27 additions and 15 deletions
+4
View File
@@ -266,6 +266,10 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
),
"Exaone4ForCausalLM": _HfExamplesInfo("LGAI-EXAONE/EXAONE-4.0-32B"),
"ExaoneMoeForCausalLM": _HfExamplesInfo("LGAI-EXAONE/K-EXAONE-236B-A23B"),
"ExaoneMoEForCausalLM": _HfExamplesInfo(
"LGAI-EXAONE/K-EXAONE-236B-A23B",
hf_overrides={"architectures": ["ExaoneMoEForCausalLM"]},
),
"FalconForCausalLM": _HfExamplesInfo("tiiuae/falcon-7b"),
"FalconH1ForCausalLM": _HfExamplesInfo("tiiuae/Falcon-H1-0.5B-Base"),
"FlexOlmoForCausalLM": _HfExamplesInfo("allenai/Flex-reddit-2x7B-1T"),
+22 -15
View File
@@ -227,24 +227,31 @@ class ExaoneMoeAttention(nn.Module):
layer_idx = extract_layer_index(prefix)
if config.sliding_windows is not None:
self.sliding_window_size = config.sliding_windows[layer_idx]
if config.layer_types[layer_idx] == "full_attention":
self.sliding_window_size = None
# Configs published before K-EXAONE 2.0 only have the scalar
# `sliding_window` and no MTP fields; their MTP layer follows the main
# layer with the same index.
layer_types = config.layer_types
sliding_windows = getattr(config, "sliding_windows", None)
if is_mtp:
self.sliding_window = (
config.mtp_layer_types[layer_idx] == "sliding_attention"
layer_types = getattr(config, "mtp_layer_types", None) or layer_types
sliding_windows = (
getattr(config, "mtp_sliding_windows", None) or sliding_windows
)
if config.mtp_sliding_windows is not None:
self.sliding_window_size = config.mtp_sliding_windows[layer_idx]
if config.mtp_layer_types[layer_idx] == "full_attention":
self.sliding_window_size = None
if layer_types[layer_idx] == "full_attention":
self.sliding_window_size = None
elif sliding_windows is not None:
self.sliding_window_size = sliding_windows[layer_idx]
else:
self.sliding_window_size = config.sliding_window
# apply rotary embeddings to every layer in full attention models
self.apply_rope_all_layers = "sliding_attention" not in config.layer_types
# Full-attention layers are NoPE when sliding layers are present, except
# the MTP layer, which is trained with RoPE.
self.use_rope = (
bool(self.sliding_window_size)
or is_mtp
or "sliding_attention" not in config.layer_types
)
set_default_rope_theta(config, default_theta=1000000)
self.rotary_emb = get_rope(
@@ -279,7 +286,7 @@ class ExaoneMoeAttention(nn.Module):
k = self.k_norm(k)
k = k.flatten(-2, -1)
if self.sliding_window_size or self.apply_rope_all_layers:
if self.use_rope:
q, k = self.rotary_emb(positions, q, k)
attn_output = self.attn(q, k, v)
output, _ = self.o_proj(attn_output)
+1
View File
@@ -98,6 +98,7 @@ _TEXT_GENERATION_MODELS = {
"ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
"Exaone4ForCausalLM": ("exaone4", "Exaone4ForCausalLM"),
"ExaoneMoeForCausalLM": ("exaone_moe", "ExaoneMoeForCausalLM"),
"ExaoneMoEForCausalLM": ("exaone_moe", "ExaoneMoeForCausalLM"),
"FalconForCausalLM": ("falcon", "FalconForCausalLM"),
"FalconMambaForCausalLM": ("mamba", "MambaForCausalLM"),
"FalconH1ForCausalLM": ("falcon_h1", "FalconH1ForCausalLM"),