[MM][Perf][CG] Support ViT full CUDA graph for Kimi-VL (#41992)
Signed-off-by: oguz <oguzhankir17@gmail.com>
This commit is contained in:
@@ -48,6 +48,13 @@ def internvl_chat_template(content: str) -> str:
|
||||
return f"<|im_start|>user\n{content}<|im_end|>\n<|im_start|>assistant\n"
|
||||
|
||||
|
||||
def kimi_vl_chat_template(content: str) -> str:
|
||||
return (
|
||||
f"<|im_user|>user<|im_middle|>{content}<|im_end|>"
|
||||
"<|im_assistant|>assistant<|im_middle|>"
|
||||
)
|
||||
|
||||
|
||||
def step3_vl_chat_template(content: str) -> str:
|
||||
return (
|
||||
"<|begin▁of▁sentence|> You are a helpful assistant.<|BOT|>user\n "
|
||||
@@ -100,6 +107,34 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
|
||||
needs_video_metadata=False,
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"kimi_vl": VitCudagraphTestConfig(
|
||||
model="moonshotai/Kimi-VL-A3B-Instruct",
|
||||
modalities=["image"],
|
||||
image_prompt=kimi_vl_chat_template(
|
||||
"<|media_start|>image<|media_content|><|media_pad|><|media_end|>"
|
||||
"What is in this image?"
|
||||
),
|
||||
needs_video_metadata=False,
|
||||
# Single bucket sized to cover the test images' output tokens.
|
||||
# The default auto-inferred range fans out into multiple power-of-2
|
||||
# buckets, each holding a full ViT capture pool.
|
||||
compilation_config_overrides={
|
||||
"encoder_cudagraph_token_budgets": [1024],
|
||||
},
|
||||
# Shrink to 1 text + 1 vision layer with random weights so the
|
||||
# test runs on any CI GPU (incl. L4) and skips the multi-GiB
|
||||
# weight download. The test only validates that encoder CG
|
||||
# capture/replay functions correctly, not output quality.
|
||||
vllm_runner_kwargs={
|
||||
"trust_remote_code": True,
|
||||
"load_format": "dummy",
|
||||
"hf_overrides": partial(
|
||||
dummy_hf_overrides,
|
||||
model_arch="KimiVLForConditionalGeneration",
|
||||
),
|
||||
},
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"qwen3_vl": VitCudagraphTestConfig(
|
||||
model="Qwen/Qwen3-VL-2B-Instruct",
|
||||
image_prompt=qwen_vl_chat_template(
|
||||
|
||||
Reference in New Issue
Block a user