[MM][Perf][CG] Support ViT full cudagraphs for mllama4 (#40660)
Signed-off-by: allgather <all2allops@gmail.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
@@ -55,6 +55,26 @@ def step3_vl_chat_template(content: str) -> str:
|
||||
|
||||
|
||||
MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
|
||||
"llama4": VitCudagraphTestConfig(
|
||||
model="meta-llama/Llama-4-Scout-17B-16E-Instruct",
|
||||
modalities=["image"],
|
||||
image_prompt=(
|
||||
"<|begin_of_text|><|header_start|>user<|header_end|>\n\n"
|
||||
"<|image|>What is in this image?<|eot|>"
|
||||
"<|header_start|>assistant<|header_end|>\n\n"
|
||||
),
|
||||
max_model_len=4096,
|
||||
max_tokens=32,
|
||||
max_num_seqs=2,
|
||||
vllm_runner_kwargs={
|
||||
"load_format": "dummy",
|
||||
"hf_overrides": partial(
|
||||
dummy_hf_overrides,
|
||||
model_arch="Llama4ForConditionalGeneration",
|
||||
),
|
||||
},
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"internvl": VitCudagraphTestConfig(
|
||||
model="OpenGVLab/InternVL3-1B",
|
||||
num_video_frames=8,
|
||||
|
||||
@@ -507,12 +507,13 @@ def dummy_hf_overrides(
|
||||
# Only set MoE related config when the model has MoE layers.
|
||||
# Otherwise all models detected as MoE by _get_transformers_backend_cls.
|
||||
if model_arch_config.num_experts > 0:
|
||||
num_experts_per_tok = 1 if model_arch == "Llama4ForConditionalGeneration" else 2
|
||||
update_dict.update(
|
||||
{
|
||||
"num_experts": num_experts,
|
||||
"num_experts_per_tok": 2,
|
||||
"num_experts_per_tok": num_experts_per_tok,
|
||||
# Kimi uses `num_experts_per_token`.
|
||||
"num_experts_per_token": 2,
|
||||
"num_experts_per_token": num_experts_per_tok,
|
||||
"num_local_experts": num_experts,
|
||||
# Otherwise there will not be any expert layers
|
||||
"first_k_dense_replace": 0,
|
||||
|
||||
Reference in New Issue
Block a user