[Bugfix][Model] K-EXAONE: restore loading of pre-2.0 configs and apply RoPE to the MTP layer (#60187)
Signed-off-by: godmook <cmoh4135@naver.com> Signed-off-by: Mook <68294499+Godmook@users.noreply.github.com>
This commit is contained in:
@@ -266,6 +266,10 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
),
|
||||
"Exaone4ForCausalLM": _HfExamplesInfo("LGAI-EXAONE/EXAONE-4.0-32B"),
|
||||
"ExaoneMoeForCausalLM": _HfExamplesInfo("LGAI-EXAONE/K-EXAONE-236B-A23B"),
|
||||
"ExaoneMoEForCausalLM": _HfExamplesInfo(
|
||||
"LGAI-EXAONE/K-EXAONE-236B-A23B",
|
||||
hf_overrides={"architectures": ["ExaoneMoEForCausalLM"]},
|
||||
),
|
||||
"FalconForCausalLM": _HfExamplesInfo("tiiuae/falcon-7b"),
|
||||
"FalconH1ForCausalLM": _HfExamplesInfo("tiiuae/Falcon-H1-0.5B-Base"),
|
||||
"FlexOlmoForCausalLM": _HfExamplesInfo("allenai/Flex-reddit-2x7B-1T"),
|
||||
|
||||
@@ -227,24 +227,31 @@ class ExaoneMoeAttention(nn.Module):
|
||||
|
||||
layer_idx = extract_layer_index(prefix)
|
||||
|
||||
if config.sliding_windows is not None:
|
||||
self.sliding_window_size = config.sliding_windows[layer_idx]
|
||||
|
||||
if config.layer_types[layer_idx] == "full_attention":
|
||||
self.sliding_window_size = None
|
||||
|
||||
# Configs published before K-EXAONE 2.0 only have the scalar
|
||||
# `sliding_window` and no MTP fields; their MTP layer follows the main
|
||||
# layer with the same index.
|
||||
layer_types = config.layer_types
|
||||
sliding_windows = getattr(config, "sliding_windows", None)
|
||||
if is_mtp:
|
||||
self.sliding_window = (
|
||||
config.mtp_layer_types[layer_idx] == "sliding_attention"
|
||||
layer_types = getattr(config, "mtp_layer_types", None) or layer_types
|
||||
sliding_windows = (
|
||||
getattr(config, "mtp_sliding_windows", None) or sliding_windows
|
||||
)
|
||||
if config.mtp_sliding_windows is not None:
|
||||
self.sliding_window_size = config.mtp_sliding_windows[layer_idx]
|
||||
|
||||
if config.mtp_layer_types[layer_idx] == "full_attention":
|
||||
self.sliding_window_size = None
|
||||
if layer_types[layer_idx] == "full_attention":
|
||||
self.sliding_window_size = None
|
||||
elif sliding_windows is not None:
|
||||
self.sliding_window_size = sliding_windows[layer_idx]
|
||||
else:
|
||||
self.sliding_window_size = config.sliding_window
|
||||
|
||||
# apply rotary embeddings to every layer in full attention models
|
||||
self.apply_rope_all_layers = "sliding_attention" not in config.layer_types
|
||||
# Full-attention layers are NoPE when sliding layers are present, except
|
||||
# the MTP layer, which is trained with RoPE.
|
||||
self.use_rope = (
|
||||
bool(self.sliding_window_size)
|
||||
or is_mtp
|
||||
or "sliding_attention" not in config.layer_types
|
||||
)
|
||||
|
||||
set_default_rope_theta(config, default_theta=1000000)
|
||||
self.rotary_emb = get_rope(
|
||||
@@ -279,7 +286,7 @@ class ExaoneMoeAttention(nn.Module):
|
||||
k = self.k_norm(k)
|
||||
k = k.flatten(-2, -1)
|
||||
|
||||
if self.sliding_window_size or self.apply_rope_all_layers:
|
||||
if self.use_rope:
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
attn_output = self.attn(q, k, v)
|
||||
output, _ = self.o_proj(attn_output)
|
||||
|
||||
@@ -98,6 +98,7 @@ _TEXT_GENERATION_MODELS = {
|
||||
"ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
|
||||
"Exaone4ForCausalLM": ("exaone4", "Exaone4ForCausalLM"),
|
||||
"ExaoneMoeForCausalLM": ("exaone_moe", "ExaoneMoeForCausalLM"),
|
||||
"ExaoneMoEForCausalLM": ("exaone_moe", "ExaoneMoeForCausalLM"),
|
||||
"FalconForCausalLM": ("falcon", "FalconForCausalLM"),
|
||||
"FalconMambaForCausalLM": ("mamba", "MambaForCausalLM"),
|
||||
"FalconH1ForCausalLM": ("falcon_h1", "FalconH1ForCausalLM"),
|
||||
|
||||
Reference in New Issue
Block a user