[Perf] FP8 FlashInfer Attn for ViT (#38065)

Signed-off-by: Zhanda Zhu <zhandazhu@gmail.com>
Co-authored-by: Yubo Gao <ybgao-nvidia@users.noreply.github.com>
This commit is contained in:
Zhanda Zhu
2026-04-27 13:44:15 +08:00
committed by GitHub
co-authored by Yubo Gao
parent 592ae6805c
commit 5d5c776444
18 changed files with 1829 additions and 49 deletions
+28
View File
@@ -542,6 +542,14 @@ class EngineArgs:
mm_encoder_attn_backend: AttentionBackendEnum | str | None = (
MultiModalConfig.mm_encoder_attn_backend
)
mm_encoder_attn_dtype: str | None = MultiModalConfig.mm_encoder_attn_dtype
mm_encoder_fp8_scale_path: str | None = MultiModalConfig.mm_encoder_fp8_scale_path
mm_encoder_fp8_scale_save_path: str | None = (
MultiModalConfig.mm_encoder_fp8_scale_save_path
)
mm_encoder_fp8_scale_save_margin: float = (
MultiModalConfig.mm_encoder_fp8_scale_save_margin
)
io_processor_plugin: str | None = None
renderer_num_workers: int = 1
skip_mm_profiling: bool = MultiModalConfig.skip_mm_profiling
@@ -1179,6 +1187,22 @@ class EngineArgs:
"--mm-encoder-attn-backend",
**multimodal_kwargs["mm_encoder_attn_backend"],
)
multimodal_group.add_argument(
"--mm-encoder-attn-dtype",
**multimodal_kwargs["mm_encoder_attn_dtype"],
)
multimodal_group.add_argument(
"--mm-encoder-fp8-scale-path",
**multimodal_kwargs["mm_encoder_fp8_scale_path"],
)
multimodal_group.add_argument(
"--mm-encoder-fp8-scale-save-path",
**multimodal_kwargs["mm_encoder_fp8_scale_save_path"],
)
multimodal_group.add_argument(
"--mm-encoder-fp8-scale-save-margin",
**multimodal_kwargs["mm_encoder_fp8_scale_save_margin"],
)
multimodal_group.add_argument(
"--interleave-mm-strings", **multimodal_kwargs["interleave_mm_strings"]
)
@@ -1517,6 +1541,10 @@ class EngineArgs:
mm_encoder_only=self.mm_encoder_only,
mm_encoder_tp_mode=self.mm_encoder_tp_mode,
mm_encoder_attn_backend=self.mm_encoder_attn_backend,
mm_encoder_attn_dtype=self.mm_encoder_attn_dtype,
mm_encoder_fp8_scale_path=self.mm_encoder_fp8_scale_path,
mm_encoder_fp8_scale_save_path=self.mm_encoder_fp8_scale_save_path,
mm_encoder_fp8_scale_save_margin=self.mm_encoder_fp8_scale_save_margin,
pooler_config=self.pooler_config,
generation_config=self.generation_config,
override_generation_config=self.override_generation_config,