[ROCm][MoE] Split AiterExperts by weight-quantization key (#60322)

Signed-off-by: Divakar Verma <divakar.verma@amd.com>
Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Divakar Verma
2026-10-13 00:13:33 -05:00
committed by GitHub
co-authored by Claude Sonnet 5
parent b337523c11
commit f7ba007b5e
13 changed files with 198 additions and 64 deletions
@@ -167,6 +167,9 @@ class Config:
vllm_config.model_config = SimpleNamespace(
enforce_eager=True,
is_moe=True,
enable_sleep_mode=False,
sleep_mode_backend="cumem",
sleep_mode_offload_cudagraph=False,
)
vllm_config.parallel_config.data_parallel_size = self.world_size
vllm_config.parallel_config.enable_expert_parallel = True
@@ -722,6 +725,9 @@ def _maybe_convert_weights_for_experts(
rank_weights: WeightTensors,
) -> WeightTensors:
"""Convert weights to expert-specific format (e.g., TrtLLM BlockMajorK)."""
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
AiterExperts,
)
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
Fp8MoeBackend,
convert_to_fp8_moe_kernel_format,
@@ -734,7 +740,7 @@ def _maybe_convert_weights_for_experts(
# intermediate sizes that are not a multiple of 256, but only for the fp8 per-tensor
# and per-token schemes. Those keep JIT-compiling until
# https://github.com/ROCm/aiter/issues/5766 is fixed.
if fe_name == "AiterExperts" and (
if issubclass(fe_type, AiterExperts) and (
config.quant_dtype is None or config.quant_block_shape is not None
):
return _shuffle_weights_for_aiter(rank_weights)
@@ -93,6 +93,7 @@ common_float_types: list[torch.dtype | str] = [
]
common_float_and_int_types = common_float_types + [torch.int8]
nvfp4_types = ["nvfp4"]
mxfp4_types = ["mxfp4"]
fp8_types = [current_platform.fp8_dtype()]
@@ -318,20 +319,37 @@ if has_flashinfer_trtllm_fused_moe() and current_platform.has_device_capability(
if has_aiter():
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
AiterExperts,
AiterFp8BlockExperts,
AiterFp8ChannelExperts,
AiterFp8TensorExperts,
AiterMxfp4Experts,
AiterUnquantizedExperts,
)
register_experts(
AiterExperts,
standard_format,
# AiterExperts also supports the fully-unquantized (None, None)
# scheme (see SUPPORTED_W_A in rocm_aiter_moe.py), not just fp8.
common_float_types,
blocked_quantization_support=True,
needs_aiter=True,
)
# Registration dtypes gate which quant configs the modular-kernel test
# harness will ever pair with a given class (see Config.is_valid()):
# they must match what each subclass's _supports_quant_scheme() actually
# declares, or the harness will generate a case the class rejects.
for aiter_experts_cls, aiter_supported_dtypes in (
(AiterUnquantizedExperts, common_float_types),
(AiterFp8BlockExperts, fp8_types),
(AiterFp8TensorExperts, fp8_types),
(AiterFp8ChannelExperts, fp8_types),
(AiterMxfp4Experts, mxfp4_types),
):
register_experts(
aiter_experts_cls,
standard_format,
aiter_supported_dtypes,
blocked_quantization_support=True,
needs_aiter=True,
)
else:
AiterExperts = None
AiterUnquantizedExperts = None
AiterFp8BlockExperts = None
AiterFp8TensorExperts = None
AiterFp8ChannelExperts = None
AiterMxfp4Experts = None
if has_deep_gemm() and is_deep_gemm_supported():
register_experts(