[Quantization] Auto-disable DeepGEMM for Kolibri 1 on SM12x (#61274)

Signed-off-by: Luca Motz <luca.motz@icloud.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Luca Motz
2026-10-14 12:59:50 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 3723a1ea96
commit 2d09fdcdc2
3 changed files with 22 additions and 13 deletions
-2
View File
@@ -1547,8 +1547,6 @@ class VllmConfig:
self.quant_config.use_deep_gemm = False self.quant_config.use_deep_gemm = False
logger.warning_once( logger.warning_once(
"Auto-disabled DeepGemm for model_type=%s on Blackwell. " "Auto-disabled DeepGemm for model_type=%s on Blackwell. "
"DeepGemm E8M0 scale format causes accuracy degradation "
"for this architecture. Falling back to CUTLASS. "
"To disable DeepGemm globally, set VLLM_USE_DEEP_GEMM=0.", "To disable DeepGemm globally, set VLLM_USE_DEEP_GEMM=0.",
model_type, model_type,
) )
@@ -9,6 +9,7 @@ import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm import envs from vllm import envs
from vllm._aiter_ops import rocm_aiter_ops from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import get_current_vllm_config
from vllm.config.kernel import MoEBackend from vllm.config.kernel import MoEBackend
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.all2all_utils import ( from vllm.model_executor.layers.fused_moe.all2all_utils import (
@@ -40,6 +41,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
) )
from vllm.model_executor.utils import replace_parameter from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform from vllm.platforms import current_platform
from vllm.utils.deep_gemm import should_auto_disable_deep_gemm
from vllm.utils.math_utils import round_up from vllm.utils.math_utils import round_up
logger = init_logger(__name__) logger = init_logger(__name__)
@@ -492,6 +494,13 @@ def select_fp8_moe_backend(
return _return_or_raise( return _return_or_raise(
backend, config, weight_key, activation_key, activation_format backend, config, weight_key, activation_key, activation_format
) )
elif (model_config := get_current_vllm_config().model_config) is not None and (
should_auto_disable_deep_gemm(
getattr(model_config.hf_text_config, "model_type", None)
)
):
AVAILABLE_BACKENDS.remove(Fp8MoeBackend.DEEPGEMM)
AVAILABLE_BACKENDS.remove(Fp8MoeBackend.BATCHED_DEEPGEMM)
# Handle explicit AITER FP8 configuration. # Handle explicit AITER FP8 configuration.
if envs.is_set("VLLM_ROCM_USE_AITER") or envs.is_set("VLLM_ROCM_USE_AITER_MOE"): if envs.is_set("VLLM_ROCM_USE_AITER") or envs.is_set("VLLM_ROCM_USE_AITER_MOE"):
+13 -11
View File
@@ -24,9 +24,13 @@ from vllm.platforms import current_platform
from vllm.utils.import_utils import has_deep_gemm from vllm.utils.import_utils import has_deep_gemm
from vllm.utils.math_utils import cdiv from vllm.utils.math_utils import cdiv
_DEEPGEMM_BLACKWELL_EXCLUDED_MODEL_TYPES: set[str] = { # Model type -> Blackwell device capability families to auto-disable DeepGemm on.
"qwen3_5_text", _DEEPGEMM_BLACKWELL_EXCLUDED_MODEL_TYPES: dict[str, tuple[int, ...]] = {
"qwen3_5_moe_text", # Accuracy
"qwen3_5_text": (100, 120),
"qwen3_5_moe_text": (100, 120),
# Latency
"kolibri1": (120,),
} }
@@ -39,17 +43,15 @@ def get_paged_mqa_page_sizes() -> tuple[int, ...]:
def should_auto_disable_deep_gemm(model_type: str | None) -> bool: def should_auto_disable_deep_gemm(model_type: str | None) -> bool:
"""Check if DeepGemm should be auto-disabled for this model on Blackwell. """Check if DeepGemm should be auto-disabled for this model on Blackwell.
Returns True if the model is known to have accuracy degradation with Returns True if DeepGemm is known to hurt this model's accuracy or
DeepGemm's E8M0 scale format on Blackwell GPUs (SM100+). latency on the current Blackwell GPU family.
""" """
if model_type is None: if model_type is None:
return False return False
if not ( return any(
current_platform.is_device_capability_family(100) current_platform.is_device_capability_family(family)
or current_platform.is_device_capability_family(120) for family in _DEEPGEMM_BLACKWELL_EXCLUDED_MODEL_TYPES.get(model_type, ())
): )
return False
return model_type in _DEEPGEMM_BLACKWELL_EXCLUDED_MODEL_TYPES
class DeepGemmQuantScaleFMT(Enum): class DeepGemmQuantScaleFMT(Enum):