From 917fdb5bf7d95c6e2ec321924e7d4620759e9674 Mon Sep 17 00:00:00 2001 From: Michael Goin Date: Wed, 22 Jul 2026 19:28:49 -0400 Subject: [PATCH] [Bugfix] Fix DeepGEMM warmup when using `FlashInferFp8DeepGEMMDynamicBlockScaledKernel` (#49467) Signed-off-by: mgoin --- vllm/model_executor/warmup/deep_gemm_warmup.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/vllm/model_executor/warmup/deep_gemm_warmup.py b/vllm/model_executor/warmup/deep_gemm_warmup.py index 2c9182e861..4b05cad1b1 100644 --- a/vllm/model_executor/warmup/deep_gemm_warmup.py +++ b/vllm/model_executor/warmup/deep_gemm_warmup.py @@ -133,6 +133,18 @@ def _extract_data_from_fused_moe_module( return w13, w13_s, w2, w2_s, num_topk +def _is_deep_gemm_backed_kernel(fp8_linear: object) -> bool: + """ + Return True if the selected linear kernel dispatches to DeepGEMM, either + directly or as the fallback branch of a dynamic wrapper. + """ + if isinstance(fp8_linear, DeepGemmFp8BlockScaledMMKernel): + return True + return isinstance( + getattr(fp8_linear, "fallback", None), DeepGemmFp8BlockScaledMMKernel + ) + + def _fp8_linear_may_use_deep_gemm(module: torch.nn.Module) -> bool: """ Return True if the input module/layer could be processed with DeepGEMM. @@ -147,10 +159,8 @@ def _fp8_linear_may_use_deep_gemm(module: torch.nn.Module) -> bool: ): return False - if not isinstance( - getattr(module.quant_method, "fp8_linear", None), - DeepGemmFp8BlockScaledMMKernel, - ): + fp8_linear = getattr(module.quant_method, "fp8_linear", None) + if not _is_deep_gemm_backed_kernel(fp8_linear): return False block_size = get_mk_alignment_for_contiguous_layout()[0]