[Quantization] Auto-disable DeepGEMM for Kolibri 1 on SM12x (#61274)

Signed-off-by: Luca Motz <luca.motz@icloud.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Luca Motz
2026-10-14 12:59:50 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 3723a1ea96
commit 2d09fdcdc2
3 changed files with 22 additions and 13 deletions
+13 -11
View File
@@ -24,9 +24,13 @@ from vllm.platforms import current_platform
from vllm.utils.import_utils import has_deep_gemm
from vllm.utils.math_utils import cdiv
_DEEPGEMM_BLACKWELL_EXCLUDED_MODEL_TYPES: set[str] = {
"qwen3_5_text",
"qwen3_5_moe_text",
# Model type -> Blackwell device capability families to auto-disable DeepGemm on.
_DEEPGEMM_BLACKWELL_EXCLUDED_MODEL_TYPES: dict[str, tuple[int, ...]] = {
# Accuracy
"qwen3_5_text": (100, 120),
"qwen3_5_moe_text": (100, 120),
# Latency
"kolibri1": (120,),
}
@@ -39,17 +43,15 @@ def get_paged_mqa_page_sizes() -> tuple[int, ...]:
def should_auto_disable_deep_gemm(model_type: str | None) -> bool:
"""Check if DeepGemm should be auto-disabled for this model on Blackwell.
Returns True if the model is known to have accuracy degradation with
DeepGemm's E8M0 scale format on Blackwell GPUs (SM100+).
Returns True if DeepGemm is known to hurt this model's accuracy or
latency on the current Blackwell GPU family.
"""
if model_type is None:
return False
if not (
current_platform.is_device_capability_family(100)
or current_platform.is_device_capability_family(120)
):
return False
return model_type in _DEEPGEMM_BLACKWELL_EXCLUDED_MODEL_TYPES
return any(
current_platform.is_device_capability_family(family)
for family in _DEEPGEMM_BLACKWELL_EXCLUDED_MODEL_TYPES.get(model_type, ())
)
class DeepGemmQuantScaleFMT(Enum):