[Perf][Kernels] Enable FlashInfer DeepGEMM swapAB on SM90 (for W8A8 Linear Op) (#29213)
Signed-off-by: Kate Cheng <yunhsuanc@nvidia.com> Signed-off-by: Jhao-Ting Chen <jhaotingc@nvidia.com> Co-authored-by: Jhao-Ting Chen <jhaotingc@nvidia.com>
This commit is contained in:
co-authored by
Jhao-Ting Chen
parent
1ab055efe6
commit
cc6dafaef2
@@ -168,6 +168,7 @@ if TYPE_CHECKING:
|
||||
"relax",
|
||||
] = "relax"
|
||||
VLLM_USE_FUSED_MOE_GROUPED_TOPK: bool = True
|
||||
VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER: bool = False
|
||||
VLLM_USE_FLASHINFER_MOE_FP16: bool = False
|
||||
VLLM_USE_FLASHINFER_MOE_FP8: bool = False
|
||||
VLLM_USE_FLASHINFER_MOE_FP4: bool = False
|
||||
@@ -1206,6 +1207,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_USE_FUSED_MOE_GROUPED_TOPK": lambda: bool(
|
||||
int(os.getenv("VLLM_USE_FUSED_MOE_GROUPED_TOPK", "1"))
|
||||
),
|
||||
# Allow use of FlashInfer FP8 block-scale GEMM for linear layers.
|
||||
# This uses TensorRT-LLM kernels and requires SM90+ (Hopper).
|
||||
"VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": lambda: bool(
|
||||
int(os.getenv("VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER", "0"))
|
||||
),
|
||||
# Allow use of FlashInfer MoE kernels for fused moe ops.
|
||||
"VLLM_USE_FLASHINFER_MOE_FP16": lambda: bool(
|
||||
int(os.getenv("VLLM_USE_FLASHINFER_MOE_FP16", "0"))
|
||||
|
||||
Reference in New Issue
Block a user