[ROCm][MoE] Split AiterExperts by weight-quantization key (#60322)
Signed-off-by: Divakar Verma <divakar.verma@amd.com> Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
b337523c11
commit
f7ba007b5e
@@ -106,7 +106,11 @@ if HAS_TRITON:
|
||||
BatchedTritonExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
AiterExperts,
|
||||
AiterFp8BlockExperts,
|
||||
AiterFp8ChannelExperts,
|
||||
AiterFp8TensorExperts,
|
||||
AiterMxfp4Experts,
|
||||
AiterUnquantizedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import (
|
||||
TritonOrDeepGemmExperts,
|
||||
@@ -132,7 +136,11 @@ if HAS_TRITON:
|
||||
)
|
||||
|
||||
__all__ += [
|
||||
"AiterExperts",
|
||||
"AiterUnquantizedExperts",
|
||||
"AiterFp8BlockExperts",
|
||||
"AiterFp8TensorExperts",
|
||||
"AiterFp8ChannelExperts",
|
||||
"AiterMxfp4Experts",
|
||||
"fused_topk",
|
||||
"fused_experts",
|
||||
"get_config_file_name",
|
||||
|
||||
@@ -482,29 +482,6 @@ class AiterExperts(mk.FusedMoEExpertsModular):
|
||||
def _supports_no_act_and_mul() -> bool:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
SUPPORTED_W_A = [
|
||||
(None, None),
|
||||
(kFp8Static128BlockSym, kFp8Dynamic128Sym),
|
||||
(kFp8StaticTensorSym, kFp8StaticTensorSym),
|
||||
(kFp8StaticTensorSym, kFp8DynamicTensorSym),
|
||||
(kFp8StaticChannelSym, kFp8DynamicTokenSym),
|
||||
(kMxfp4Static, None),
|
||||
(kMxfp4Static, kMxfp4Dynamic),
|
||||
]
|
||||
if (weight_key, activation_key) not in SUPPORTED_W_A:
|
||||
return False
|
||||
if weight_key == kMxfp4Static:
|
||||
from vllm.platforms.rocm import on_gfx950, on_gfx1250
|
||||
|
||||
if not on_gfx950() or on_gfx1250():
|
||||
return False
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return activation in [
|
||||
@@ -597,3 +574,77 @@ class AiterExperts(mk.FusedMoEExpertsModular):
|
||||
output.set_(result)
|
||||
else:
|
||||
output.copy_(result)
|
||||
|
||||
|
||||
class AiterUnquantizedExperts(AiterExperts):
|
||||
"""Unquantized (bf16/fp16) weights and activations."""
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
SUPPORTED_W_A = [(None, None)]
|
||||
return (weight_key, activation_key) in SUPPORTED_W_A
|
||||
|
||||
|
||||
class AiterFp8BlockExperts(AiterExperts):
|
||||
"""128x128 block-quantized weight scale, 128-grouped dynamic
|
||||
activation scale."""
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
SUPPORTED_W_A = [(kFp8Static128BlockSym, kFp8Dynamic128Sym)]
|
||||
return (weight_key, activation_key) in SUPPORTED_W_A
|
||||
|
||||
|
||||
class AiterFp8TensorExperts(AiterExperts):
|
||||
"""Per-tensor static weight scale, either per-tensor static or
|
||||
per-tensor dynamic activation scale."""
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
SUPPORTED_W_A = [
|
||||
(kFp8StaticTensorSym, kFp8StaticTensorSym),
|
||||
(kFp8StaticTensorSym, kFp8DynamicTensorSym),
|
||||
]
|
||||
return (weight_key, activation_key) in SUPPORTED_W_A
|
||||
|
||||
|
||||
class AiterFp8ChannelExperts(AiterExperts):
|
||||
"""Per-output-channel static weight scale, per-token dynamic
|
||||
activation scale."""
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
SUPPORTED_W_A = [(kFp8StaticChannelSym, kFp8DynamicTokenSym)]
|
||||
return (weight_key, activation_key) in SUPPORTED_W_A
|
||||
|
||||
|
||||
class AiterMxfp4Experts(AiterExperts):
|
||||
"""MXFP4-quantized weights, with either unquantized (bf16/fp16) or
|
||||
MXFP4-quantized dynamic activations. gfx950-only (not gfx1250)."""
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
SUPPORTED_W_A = [
|
||||
(kMxfp4Static, None),
|
||||
(kMxfp4Static, kMxfp4Dynamic),
|
||||
]
|
||||
if (weight_key, activation_key) not in SUPPORTED_W_A:
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx950, on_gfx1250
|
||||
|
||||
return on_gfx950() and not on_gfx1250()
|
||||
|
||||
@@ -211,10 +211,12 @@ def backend_to_kernel_cls(
|
||||
|
||||
elif backend == Fp8MoeBackend.AITER:
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
AiterExperts,
|
||||
AiterFp8BlockExperts,
|
||||
AiterFp8ChannelExperts,
|
||||
AiterFp8TensorExperts,
|
||||
)
|
||||
|
||||
return [AiterExperts]
|
||||
return [AiterFp8BlockExperts, AiterFp8TensorExperts, AiterFp8ChannelExperts]
|
||||
|
||||
elif backend == Fp8MoeBackend.VLLM_CUTLASS:
|
||||
from vllm.model_executor.layers.fused_moe.experts.triton_cutlass_moe import (
|
||||
|
||||
@@ -284,10 +284,10 @@ def backend_to_kernel_cls(
|
||||
|
||||
elif backend == Mxfp4MoeBackend.AITER_MXFP4_BF16:
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
AiterExperts,
|
||||
AiterMxfp4Experts,
|
||||
)
|
||||
|
||||
return [AiterExperts]
|
||||
return [AiterMxfp4Experts]
|
||||
|
||||
elif backend == Mxfp4MoeBackend.AITER_TRITON_MXFP4_BF16:
|
||||
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a16_moe import (
|
||||
@@ -305,10 +305,10 @@ def backend_to_kernel_cls(
|
||||
|
||||
elif backend == Mxfp4MoeBackend.AITER_MXFP4_MXFP4:
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
AiterExperts,
|
||||
AiterMxfp4Experts,
|
||||
)
|
||||
|
||||
return [AiterExperts]
|
||||
return [AiterMxfp4Experts]
|
||||
|
||||
elif backend == Mxfp4MoeBackend.XPU:
|
||||
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import XPUExpertsMxFp4
|
||||
|
||||
@@ -126,10 +126,10 @@ def backend_to_kernel_cls(
|
||||
|
||||
elif backend == UnquantizedMoeBackend.AITER:
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
AiterExperts,
|
||||
AiterUnquantizedExperts,
|
||||
)
|
||||
|
||||
return [AiterExperts]
|
||||
return [AiterUnquantizedExperts]
|
||||
|
||||
elif backend == UnquantizedMoeBackend.TRITON:
|
||||
from vllm.model_executor.layers.fused_moe.experts.triton_moe import (
|
||||
|
||||
Reference in New Issue
Block a user