[Perf] FP8 FlashInfer Attn for ViT (#38065)

Signed-off-by: Zhanda Zhu <zhandazhu@gmail.com>
Co-authored-by: Yubo Gao <ybgao-nvidia@users.noreply.github.com>
This commit is contained in:
Zhanda Zhu
2026-04-27 13:44:15 +08:00
committed by GitHub
co-authored by Yubo Gao
parent 592ae6805c
commit 5d5c776444
18 changed files with 1829 additions and 49 deletions
+12
View File
@@ -326,6 +326,10 @@ class ModelConfig:
mm_encoder_only: InitVar[bool | None] = None
mm_encoder_tp_mode: InitVar[MMEncoderTPMode | None] = None
mm_encoder_attn_backend: InitVar[AttentionBackendEnum | str | None] = None
mm_encoder_attn_dtype: InitVar[str | None] = None
mm_encoder_fp8_scale_path: InitVar[str | None] = None
mm_encoder_fp8_scale_save_path: InitVar[str | None] = None
mm_encoder_fp8_scale_save_margin: InitVar[float | None] = None
interleave_mm_strings: InitVar[bool | None] = None
skip_mm_profiling: InitVar[bool | None] = None
video_pruning_rate: InitVar[float | None] = None
@@ -447,6 +451,10 @@ class ModelConfig:
mm_encoder_only: bool | None,
mm_encoder_tp_mode: MMEncoderTPMode | None,
mm_encoder_attn_backend: AttentionBackendEnum | str | None,
mm_encoder_attn_dtype: str | None,
mm_encoder_fp8_scale_path: str | None,
mm_encoder_fp8_scale_save_path: str | None,
mm_encoder_fp8_scale_save_margin: float | None,
interleave_mm_strings: bool | None,
skip_mm_profiling: bool | None,
video_pruning_rate: float | None,
@@ -643,6 +651,10 @@ class ModelConfig:
mm_encoder_only=mm_encoder_only,
mm_encoder_tp_mode=mm_encoder_tp_mode,
mm_encoder_attn_backend=mm_encoder_attn_backend,
mm_encoder_attn_dtype=mm_encoder_attn_dtype,
mm_encoder_fp8_scale_path=mm_encoder_fp8_scale_path,
mm_encoder_fp8_scale_save_path=mm_encoder_fp8_scale_save_path,
mm_encoder_fp8_scale_save_margin=mm_encoder_fp8_scale_save_margin,
interleave_mm_strings=interleave_mm_strings,
skip_mm_profiling=skip_mm_profiling,
video_pruning_rate=video_pruning_rate,