[Perf] FP8 FlashInfer Attn for ViT (#38065)
Signed-off-by: Zhanda Zhu <zhandazhu@gmail.com> Co-authored-by: Yubo Gao <ybgao-nvidia@users.noreply.github.com>
This commit is contained in:
@@ -326,6 +326,10 @@ class ModelConfig:
|
||||
mm_encoder_only: InitVar[bool | None] = None
|
||||
mm_encoder_tp_mode: InitVar[MMEncoderTPMode | None] = None
|
||||
mm_encoder_attn_backend: InitVar[AttentionBackendEnum | str | None] = None
|
||||
mm_encoder_attn_dtype: InitVar[str | None] = None
|
||||
mm_encoder_fp8_scale_path: InitVar[str | None] = None
|
||||
mm_encoder_fp8_scale_save_path: InitVar[str | None] = None
|
||||
mm_encoder_fp8_scale_save_margin: InitVar[float | None] = None
|
||||
interleave_mm_strings: InitVar[bool | None] = None
|
||||
skip_mm_profiling: InitVar[bool | None] = None
|
||||
video_pruning_rate: InitVar[float | None] = None
|
||||
@@ -447,6 +451,10 @@ class ModelConfig:
|
||||
mm_encoder_only: bool | None,
|
||||
mm_encoder_tp_mode: MMEncoderTPMode | None,
|
||||
mm_encoder_attn_backend: AttentionBackendEnum | str | None,
|
||||
mm_encoder_attn_dtype: str | None,
|
||||
mm_encoder_fp8_scale_path: str | None,
|
||||
mm_encoder_fp8_scale_save_path: str | None,
|
||||
mm_encoder_fp8_scale_save_margin: float | None,
|
||||
interleave_mm_strings: bool | None,
|
||||
skip_mm_profiling: bool | None,
|
||||
video_pruning_rate: float | None,
|
||||
@@ -643,6 +651,10 @@ class ModelConfig:
|
||||
mm_encoder_only=mm_encoder_only,
|
||||
mm_encoder_tp_mode=mm_encoder_tp_mode,
|
||||
mm_encoder_attn_backend=mm_encoder_attn_backend,
|
||||
mm_encoder_attn_dtype=mm_encoder_attn_dtype,
|
||||
mm_encoder_fp8_scale_path=mm_encoder_fp8_scale_path,
|
||||
mm_encoder_fp8_scale_save_path=mm_encoder_fp8_scale_save_path,
|
||||
mm_encoder_fp8_scale_save_margin=mm_encoder_fp8_scale_save_margin,
|
||||
interleave_mm_strings=interleave_mm_strings,
|
||||
skip_mm_profiling=skip_mm_profiling,
|
||||
video_pruning_rate=video_pruning_rate,
|
||||
|
||||
Reference in New Issue
Block a user