[Bugfix] Support HF-config compat for Inkling (#51850)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Michael Goin
2026-08-11 15:32:28 -04:00
committed by GitHub
parent b2ab096017
commit ded6c452c3
+19 -1
View File
@@ -22,6 +22,7 @@ class InklingModelConfig(PretrainedConfig):
hidden_size: int = 1536,
intermediate_size: int = 768,
dense_intermediate_size: int | None = None,
moe_intermediate_size: int | None = None,
num_hidden_layers: int = 16,
num_attention_heads: int = 12,
num_key_value_heads: int = 4,
@@ -41,7 +42,8 @@ class InklingModelConfig(PretrainedConfig):
o_bias: bool = False,
use_embed_norm: bool = False,
use_sconv: bool = False,
sconv_kernel_size: int = 4,
sconv_kernel_size: int | None = None,
conv_kernel_size: int | None = None,
dense_mlp_idx: int = 0,
n_routed_experts: int = 0,
n_shared_experts: int = 0,
@@ -77,8 +79,24 @@ class InklingModelConfig(PretrainedConfig):
swa_head_dim = head_dim
if swa_v_head_dim is None:
swa_v_head_dim = swa_head_dim
# OG format: https://huggingface.co/thinkingmachines/Inkling
# HF format: https://github.com/huggingface/transformers/blob/fe747d88a3296bd94d426db2717f232f9d4afdb7/src/transformers/models/inkling/configuration_inkling.py#L95
# Huggingface Transformers compatibility:
# HF intermediate_size == OG dense_intermediate_size
# HF moe_intermediate_size == OG intermediate_size
# HF conv_kernel_size == OG sconv_kernel_size
if (
moe_intermediate_size is not None
and intermediate_size is not None
and dense_intermediate_size is None
):
dense_intermediate_size = intermediate_size
intermediate_size = moe_intermediate_size
if dense_intermediate_size is None:
dense_intermediate_size = intermediate_size
# HF conv_kernel_size == OG sconv_kernel_size
if sconv_kernel_size is None:
sconv_kernel_size = conv_kernel_size if conv_kernel_size is not None else 4
if local_layer_ids is None:
local_layer_ids = []