[Quantization] Auto-disable DeepGEMM for Kolibri 1 on SM12x (#61274)
Signed-off-by: Luca Motz <luca.motz@icloud.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
3723a1ea96
commit
2d09fdcdc2
@@ -9,6 +9,7 @@ import torch
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm import envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.config.kernel import MoEBackend
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
@@ -40,6 +41,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import should_auto_disable_deep_gemm
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -492,6 +494,13 @@ def select_fp8_moe_backend(
|
||||
return _return_or_raise(
|
||||
backend, config, weight_key, activation_key, activation_format
|
||||
)
|
||||
elif (model_config := get_current_vllm_config().model_config) is not None and (
|
||||
should_auto_disable_deep_gemm(
|
||||
getattr(model_config.hf_text_config, "model_type", None)
|
||||
)
|
||||
):
|
||||
AVAILABLE_BACKENDS.remove(Fp8MoeBackend.DEEPGEMM)
|
||||
AVAILABLE_BACKENDS.remove(Fp8MoeBackend.BATCHED_DEEPGEMM)
|
||||
|
||||
# Handle explicit AITER FP8 configuration.
|
||||
if envs.is_set("VLLM_ROCM_USE_AITER") or envs.is_set("VLLM_ROCM_USE_AITER_MOE"):
|
||||
|
||||
Reference in New Issue
Block a user