[MM][Perf][CG] Support ViT full cudagraphs for mllama4 (#40660)

Signed-off-by: allgather <all2allops@gmail.com>
Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
allgather
2026-06-11 22:17:55 -07:00
committed by GitHub
co-authored by Isotr0py
parent eb28452b10
commit 39dee1114a
5 changed files with 193 additions and 14 deletions
+3 -2
View File
@@ -507,12 +507,13 @@ def dummy_hf_overrides(
# Only set MoE related config when the model has MoE layers.
# Otherwise all models detected as MoE by _get_transformers_backend_cls.
if model_arch_config.num_experts > 0:
num_experts_per_tok = 1 if model_arch == "Llama4ForConditionalGeneration" else 2
update_dict.update(
{
"num_experts": num_experts,
"num_experts_per_tok": 2,
"num_experts_per_tok": num_experts_per_tok,
# Kimi uses `num_experts_per_token`.
"num_experts_per_token": 2,
"num_experts_per_token": num_experts_per_tok,
"num_local_experts": num_experts,
# Otherwise there will not be any expert layers
"first_k_dense_replace": 0,