[ROCm][Bugfix] Fall back from CK MXFP4 MoE when GEMM dimensions are unsupported (#35893)

Signed-off-by: Li <chuali@amd.com>
This commit is contained in:
Chuan (Richard) Li
2026-03-04 08:30:54 +00:00
committed by GitHub
parent 36bf213181
commit 5dc3538736
3 changed files with 63 additions and 1 deletions
@@ -48,6 +48,7 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
prepare_moe_fp4_layer_for_marlin,
)
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
CK_MXFP4_MOE_DIM_ALIGNMENT,
_can_support_mxfp4,
_swizzle_mxfp4,
get_padding_alignment,
@@ -259,6 +260,31 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
get_current_vllm_config().compilation_config.max_cudagraph_capture_size
)
# CK's pre-compiled MXFP4 MoE GEMM kernel instances have dimension
# alignment requirements. Fall back to Triton when not met.
if (
self.mxfp4_backend == Mxfp4Backend.CK
and moe.intermediate_size_per_partition % CK_MXFP4_MOE_DIM_ALIGNMENT != 0
):
if has_triton_kernels():
logger.warning_once(
"CK MXFP4 MoE GEMM does not support "
"intermediate_size_per_partition=%d (not a multiple of "
"%d). Falling back to Triton backend.",
moe.intermediate_size_per_partition,
CK_MXFP4_MOE_DIM_ALIGNMENT,
)
self.mxfp4_backend = Mxfp4Backend.TRITON
else:
raise ValueError(
f"CK MXFP4 MoE GEMM does not support "
f"intermediate_size_per_partition="
f"{moe.intermediate_size_per_partition} (not a multiple "
f"of {CK_MXFP4_MOE_DIM_ALIGNMENT}) and no Triton "
f"fallback is available. Use a compatible "
f"tensor_parallel_size."
)
assert self.mxfp4_backend != Mxfp4Backend.NONE, (
f"get_mxfp4_backend(with_lora_support={moe.is_lora_enabled}) found"
"no compatible MXFP4 MoE backend (FlashInfer/Marlin/Triton)."