[MM][CG] Support ViT CUDA Graph for Gemma-4 (#46837)

Signed-off-by: Anthony Su <xsuanthony@gmail.com>
Co-authored-by: Linkun Chen <github@lkchen.net>
This commit is contained in:
anthonsu
2026-07-25 15:02:09 -05:00
committed by GitHub
co-authored by Linkun Chen
parent ee1d996367
commit 70009fb934
4 changed files with 476 additions and 1 deletions
@@ -503,6 +503,45 @@ def run_gemma3n(questions: list[str], modality: str) -> ModelRequestData:
)
# Gemma 4
def run_gemma4(questions: list[str], modality: str) -> ModelRequestData:
assert modality in ("image", "video")
model_name = "google/gemma-4-31B-it"
# NOTE: Gemma-4-31B is a large model. Users running into Out-Of-Memory (OOM)
# errors might need to set `tensor_parallel_size` to > 1.
engine_args = EngineArgs(
model=model_name,
max_model_len=4096,
max_num_seqs=2,
limit_mm_per_prompt={modality: 1},
)
if modality == "image":
prompts = [
(
"<bos><start_of_turn>user\n"
f"<|image|>\n{question}<end_of_turn>\n"
"<start_of_turn>model\n"
)
for question in questions
]
else: # video
prompts = [
(
"<bos><start_of_turn>user\n"
f"<|video|>\n{question}<end_of_turn>\n"
"<start_of_turn>model\n"
)
for question in questions
]
return ModelRequestData(
engine_args=engine_args,
prompts=prompts,
)
# GLM-4v
def run_glm4v(questions: list[str], modality: str) -> ModelRequestData:
assert modality == "image"
@@ -2303,6 +2342,7 @@ model_example_map = {
"exaone4_5": run_exaone4_5,
"gemma3": run_gemma3,
"gemma3n": run_gemma3n,
"gemma4": run_gemma4,
"glm4v": run_glm4v,
"glm4_1v": run_glm4_1v,
"glm4_5v": run_glm4_5v,
@@ -2374,6 +2414,7 @@ MODELS_NEED_VIDEO_METADATA = [
MODELS_SUPPORT_VIT_CUDA_GRAPH = [
"llama4",
"gemma4",
"qwen2_vl",
"qwen2_5_vl",
"qwen3_vl",