[Bug] [Spec Decode] Fix model_initialization test and mismatch in aux_hidden_layers (#24613)

Signed-off-by: wwl2755 <wangwenlong2755@gmail.com>
Signed-off-by: Roger Wang <hey@rogerw.io>
Signed-off-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
This commit is contained in:
Wenlong Wang
2025-09-10 21:23:18 -07:00
committed by GitHub
co-authored by Roger Wang Cyrus Leung
parent 55b823ba0f
commit 6c8deacd72
3 changed files with 32 additions and 11 deletions
+12 -3
View File
@@ -396,6 +396,7 @@ def dummy_hf_overrides(
*,
model_arch: str = "",
exist_overrides: Optional[dict[str, Any]] = None,
use_original_num_layers: bool = False,
) -> PretrainedConfig:
"""
Dummy HF overrides function used to create dummy model
@@ -412,10 +413,18 @@ def dummy_hf_overrides(
# we use three layers for Gemma-3n to check
# both normal layer and kv_shared_layer
num_hidden_layers = (3 if model_arch == "Gemma3nForConditionalGeneration"
else 1)
if use_original_num_layers:
# Use the original number of layers from the config
num_layers = getattr(text_config, 'num_layers', 1)
num_hidden_layers = getattr(text_config, 'num_hidden_layers', 1)
else:
# Use minimal layers for testing
num_layers = 1
num_hidden_layers = (3 if model_arch
== "Gemma3nForConditionalGeneration" else 1)
text_config.update({
"num_layers": 1,
"num_layers": num_layers,
"num_hidden_layers": num_hidden_layers,
"num_experts": num_experts,
"num_experts_per_tok": 2,