[ROCm][MoE] Split AiterExperts by weight-quantization key (#60322)
Signed-off-by: Divakar Verma <divakar.verma@amd.com> Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
b337523c11
commit
f7ba007b5e
@@ -167,6 +167,9 @@ class Config:
|
||||
vllm_config.model_config = SimpleNamespace(
|
||||
enforce_eager=True,
|
||||
is_moe=True,
|
||||
enable_sleep_mode=False,
|
||||
sleep_mode_backend="cumem",
|
||||
sleep_mode_offload_cudagraph=False,
|
||||
)
|
||||
vllm_config.parallel_config.data_parallel_size = self.world_size
|
||||
vllm_config.parallel_config.enable_expert_parallel = True
|
||||
@@ -722,6 +725,9 @@ def _maybe_convert_weights_for_experts(
|
||||
rank_weights: WeightTensors,
|
||||
) -> WeightTensors:
|
||||
"""Convert weights to expert-specific format (e.g., TrtLLM BlockMajorK)."""
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
AiterExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
Fp8MoeBackend,
|
||||
convert_to_fp8_moe_kernel_format,
|
||||
@@ -734,7 +740,7 @@ def _maybe_convert_weights_for_experts(
|
||||
# intermediate sizes that are not a multiple of 256, but only for the fp8 per-tensor
|
||||
# and per-token schemes. Those keep JIT-compiling until
|
||||
# https://github.com/ROCm/aiter/issues/5766 is fixed.
|
||||
if fe_name == "AiterExperts" and (
|
||||
if issubclass(fe_type, AiterExperts) and (
|
||||
config.quant_dtype is None or config.quant_block_shape is not None
|
||||
):
|
||||
return _shuffle_weights_for_aiter(rank_weights)
|
||||
|
||||
@@ -93,6 +93,7 @@ common_float_types: list[torch.dtype | str] = [
|
||||
]
|
||||
common_float_and_int_types = common_float_types + [torch.int8]
|
||||
nvfp4_types = ["nvfp4"]
|
||||
mxfp4_types = ["mxfp4"]
|
||||
fp8_types = [current_platform.fp8_dtype()]
|
||||
|
||||
|
||||
@@ -318,20 +319,37 @@ if has_flashinfer_trtllm_fused_moe() and current_platform.has_device_capability(
|
||||
|
||||
if has_aiter():
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
AiterExperts,
|
||||
AiterFp8BlockExperts,
|
||||
AiterFp8ChannelExperts,
|
||||
AiterFp8TensorExperts,
|
||||
AiterMxfp4Experts,
|
||||
AiterUnquantizedExperts,
|
||||
)
|
||||
|
||||
register_experts(
|
||||
AiterExperts,
|
||||
standard_format,
|
||||
# AiterExperts also supports the fully-unquantized (None, None)
|
||||
# scheme (see SUPPORTED_W_A in rocm_aiter_moe.py), not just fp8.
|
||||
common_float_types,
|
||||
blocked_quantization_support=True,
|
||||
needs_aiter=True,
|
||||
)
|
||||
# Registration dtypes gate which quant configs the modular-kernel test
|
||||
# harness will ever pair with a given class (see Config.is_valid()):
|
||||
# they must match what each subclass's _supports_quant_scheme() actually
|
||||
# declares, or the harness will generate a case the class rejects.
|
||||
for aiter_experts_cls, aiter_supported_dtypes in (
|
||||
(AiterUnquantizedExperts, common_float_types),
|
||||
(AiterFp8BlockExperts, fp8_types),
|
||||
(AiterFp8TensorExperts, fp8_types),
|
||||
(AiterFp8ChannelExperts, fp8_types),
|
||||
(AiterMxfp4Experts, mxfp4_types),
|
||||
):
|
||||
register_experts(
|
||||
aiter_experts_cls,
|
||||
standard_format,
|
||||
aiter_supported_dtypes,
|
||||
blocked_quantization_support=True,
|
||||
needs_aiter=True,
|
||||
)
|
||||
else:
|
||||
AiterExperts = None
|
||||
AiterUnquantizedExperts = None
|
||||
AiterFp8BlockExperts = None
|
||||
AiterFp8TensorExperts = None
|
||||
AiterFp8ChannelExperts = None
|
||||
AiterMxfp4Experts = None
|
||||
|
||||
if has_deep_gemm() and is_deep_gemm_supported():
|
||||
register_experts(
|
||||
|
||||
@@ -16,7 +16,13 @@ import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import AiterExperts
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
AiterExperts,
|
||||
AiterFp8BlockExperts,
|
||||
AiterFp8ChannelExperts,
|
||||
AiterFp8TensorExperts,
|
||||
AiterUnquantizedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8StaticTensorSym,
|
||||
@@ -80,10 +86,10 @@ def assert_aiter_quant_scheme_case(config: Config) -> None:
|
||||
indirectly through the general quant-config sweep.
|
||||
See https://github.com/vllm-project/vllm/issues/54966."""
|
||||
fe_cls = config.fused_experts_type
|
||||
if fe_cls is not AiterExperts:
|
||||
if not issubclass(fe_cls, AiterExperts):
|
||||
return
|
||||
|
||||
if config.quant_config is None:
|
||||
if config.quant_config is None or config.quant_dtype is None:
|
||||
w_key, a_key = None, None
|
||||
else:
|
||||
w_key, a_key = config.fp8_quant_key_pair()
|
||||
@@ -101,7 +107,7 @@ def assert_aiter_activation_case(config: Config) -> None:
|
||||
AiterExperts being reached only indirectly through the general
|
||||
activation sweep. See https://github.com/vllm-project/vllm/issues/54966."""
|
||||
fe_cls = config.fused_experts_type
|
||||
if fe_cls is not AiterExperts:
|
||||
if not issubclass(fe_cls, AiterExperts):
|
||||
return
|
||||
|
||||
assert fe_cls._supports_activation(config.activation), (
|
||||
@@ -156,7 +162,7 @@ def rank_worker(
|
||||
if (
|
||||
topk == 1
|
||||
and config.supports_apply_weight_on_input()
|
||||
and config.fused_experts_type is AiterExperts
|
||||
and issubclass(config.fused_experts_type, AiterExperts)
|
||||
and config.quant_block_shape is not None
|
||||
):
|
||||
print(
|
||||
@@ -172,7 +178,7 @@ def rank_worker(
|
||||
# https://github.com/vllm-project/vllm/issues/57029
|
||||
if (
|
||||
config.world_size > 1
|
||||
and config.fused_experts_type is AiterExperts
|
||||
and issubclass(config.fused_experts_type, AiterExperts)
|
||||
and getattr(config.prepare_finalize_type, "__name__", "")
|
||||
in ("DeepEPHTPrepareAndFinalize", "MoriPrepareAndFinalize")
|
||||
):
|
||||
@@ -210,7 +216,7 @@ def rank_worker(
|
||||
|
||||
is_aiter_fp8 = (
|
||||
_cp.is_rocm()
|
||||
and config.fused_experts_type is AiterExperts
|
||||
and issubclass(config.fused_experts_type, AiterExperts)
|
||||
and config.quant_config is not None
|
||||
)
|
||||
if is_aiter_fp8:
|
||||
@@ -270,7 +276,7 @@ def is_nyi_config(config: Config) -> bool:
|
||||
return True
|
||||
|
||||
if config.activation != MoEActivation.SILU:
|
||||
if config.fused_experts_type is not AiterExperts:
|
||||
if not issubclass(config.fused_experts_type, AiterExperts):
|
||||
return True # AITER-only for this axis, for now
|
||||
if config.quant_dtype is not None:
|
||||
return True # unquantized-only for this axis, for now
|
||||
@@ -591,7 +597,7 @@ def test_aiter_moe_sorting_backend_dispatch_env_matrix(
|
||||
dtype=torch.bfloat16,
|
||||
quant_config=None,
|
||||
prepare_finalize_type=MoEPrepareAndFinalizeNoDPEPModular,
|
||||
fused_experts_type=AiterExperts,
|
||||
fused_experts_type=AiterUnquantizedExperts,
|
||||
world_size=1,
|
||||
)
|
||||
assert config.is_valid()[0]
|
||||
@@ -671,7 +677,7 @@ def test_aiter_moe_dispatch_policy_forwarded_through_apply(dispatch_policy: int)
|
||||
dtype=torch.bfloat16,
|
||||
quant_config=None,
|
||||
prepare_finalize_type=MoEPrepareAndFinalizeNoDPEPModular,
|
||||
fused_experts_type=AiterExperts,
|
||||
fused_experts_type=AiterUnquantizedExperts,
|
||||
world_size=1,
|
||||
)
|
||||
assert config.is_valid()[0]
|
||||
@@ -738,6 +744,20 @@ _PADDING_QUANT_IDS = [
|
||||
]
|
||||
|
||||
|
||||
def _padding_matrix_fe_cls(
|
||||
quant_config: TestMoEQuantConfig | None,
|
||||
) -> type[AiterExperts]:
|
||||
"""Map a _PADDING_QUANT_CONFIGS entry to the AiterExperts leaf class that
|
||||
supports it (mirrors Config.fp8_quant_key_pair's scheme derivation)."""
|
||||
if quant_config is None:
|
||||
return AiterUnquantizedExperts
|
||||
if quant_config.block_shape is not None:
|
||||
return AiterFp8BlockExperts
|
||||
if quant_config.per_out_ch_quant:
|
||||
return AiterFp8ChannelExperts
|
||||
return AiterFp8TensorExperts
|
||||
|
||||
|
||||
def _slice_gate_up_rows(t: torch.Tensor, real_per_half: int) -> torch.Tensor:
|
||||
"""Slice a (E, 2*padded_per_half, ...) tensor's dim=1 down to the real
|
||||
gate/up halves of `real_per_half` size each."""
|
||||
@@ -950,7 +970,7 @@ def test_aiter_moe_padding_matrix(mode: str, quant_config: TestMoEQuantConfig |
|
||||
dtype=torch.bfloat16,
|
||||
quant_config=quant_config,
|
||||
prepare_finalize_type=MoEPrepareAndFinalizeNoDPEPModular,
|
||||
fused_experts_type=AiterExperts,
|
||||
fused_experts_type=_padding_matrix_fe_cls(quant_config),
|
||||
world_size=1,
|
||||
hidden_dim_unpadded=_PADDING_K_UNPADDED if pad_hidden else None,
|
||||
intermediate_size_per_partition_unpadded=(
|
||||
@@ -1113,7 +1133,7 @@ def test_aiter_moe_token_padding_garbage_rows(
|
||||
dtype=torch.bfloat16,
|
||||
quant_config=quant_config,
|
||||
prepare_finalize_type=MoEPrepareAndFinalizeNoDPEPModular,
|
||||
fused_experts_type=AiterExperts,
|
||||
fused_experts_type=_padding_matrix_fe_cls(quant_config),
|
||||
world_size=1,
|
||||
)
|
||||
assert config.is_valid()[0]
|
||||
|
||||
@@ -42,7 +42,10 @@ MNKS = [
|
||||
|
||||
def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, fp8, graph):
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import AiterExperts
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
AiterFp8ChannelExperts,
|
||||
AiterUnquantizedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.mori import (
|
||||
MoriPrepareAndFinalize,
|
||||
)
|
||||
@@ -94,7 +97,8 @@ def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, fp8, graph):
|
||||
assert isinstance(prepare, MoriPrepareAndFinalize)
|
||||
assert prepare.use_fp8_dispatch == fp8
|
||||
assert prepare.num_dispatchers() == 2
|
||||
experts = AiterExperts(config, quant)
|
||||
experts_cls = AiterFp8ChannelExperts if fp8 else AiterUnquantizedExperts
|
||||
experts = experts_cls(config, quant)
|
||||
assert not experts.expects_unquantized_inputs
|
||||
kernel = FusedMoEKernel(prepare, experts)
|
||||
local_w1, local_w2 = rocm_aiter_ops.shuffle_weights(
|
||||
|
||||
@@ -848,13 +848,13 @@ def test_activation_method_enum_values():
|
||||
def test_aiter_mxfp4_quant_scheme_support_matches_gfx950():
|
||||
"""AITER MXFP4 MoE support should stay gfx950-only."""
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
AiterExperts,
|
||||
AiterMxfp4Experts,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kMxfp4Static,
|
||||
)
|
||||
|
||||
assert AiterExperts._supports_quant_scheme(kMxfp4Static, None) is on_gfx950()
|
||||
assert AiterMxfp4Experts._supports_quant_scheme(kMxfp4Static, None) is on_gfx950()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
|
||||
@@ -87,7 +87,7 @@ def _torch_topk_softplus_sqrt(
|
||||
"backend,expected_experts",
|
||||
[
|
||||
("aiter_triton_mxfp4_bf16", "AiterW4A16ExpertsMonolithic"),
|
||||
("aiter", "AiterExperts"),
|
||||
("aiter", "AiterMxfp4Experts"),
|
||||
("triton_unfused", "UnfusedOAITritonExperts"),
|
||||
],
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user