[XPU][CT] Support mxfp8 moe model (#41918)
Signed-off-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
@@ -17,6 +17,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8StaticTensorSym,
|
||||
kMxfp4Static,
|
||||
kMxfp8Dynamic,
|
||||
kMxfp8Static,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -47,6 +49,7 @@ class XPUExperts(mk.FusedMoEExpertsModular):
|
||||
)
|
||||
self.is_fp8 = False
|
||||
self.is_mxfp4 = False
|
||||
self.is_mxfp8 = False
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
@@ -148,6 +151,7 @@ class XPUExperts(mk.FusedMoEExpertsModular):
|
||||
output=output,
|
||||
is_fp8=self.is_fp8,
|
||||
is_mxfp4=self.is_mxfp4,
|
||||
is_mxfp8=self.is_mxfp8,
|
||||
)
|
||||
|
||||
|
||||
@@ -167,6 +171,46 @@ class XPUExpertsFp8(XPUExperts):
|
||||
)
|
||||
self.is_fp8 = True
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
SUPPORTED_W_A = [
|
||||
(kFp8StaticTensorSym, None),
|
||||
(kFp8StaticTensorSym, kFp8DynamicTensorSym),
|
||||
]
|
||||
return (weight_key, activation_key) in SUPPORTED_W_A
|
||||
|
||||
|
||||
class XPUExpertsMxfp8(XPUExpertsFp8):
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
max_num_tokens: int | None = None,
|
||||
num_dispatchers: int | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
moe_config,
|
||||
quant_config,
|
||||
max_num_tokens,
|
||||
num_dispatchers,
|
||||
)
|
||||
assert quant_config.quant_dtype == "mxfp8"
|
||||
self.is_mxfp8 = True
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
SUPPORTED_W_A = [
|
||||
(kMxfp8Static, None),
|
||||
(kMxfp8Static, kMxfp8Dynamic),
|
||||
]
|
||||
return (weight_key, activation_key) in SUPPORTED_W_A
|
||||
|
||||
|
||||
class XPUExpertsMXFp4(XPUExperts):
|
||||
def __init__(
|
||||
|
||||
@@ -185,9 +185,10 @@ def backend_to_kernel_cls(
|
||||
elif backend == Fp8MoeBackend.XPU:
|
||||
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import (
|
||||
XPUExpertsFp8,
|
||||
XPUExpertsMxfp8,
|
||||
)
|
||||
|
||||
return [XPUExpertsFp8]
|
||||
return [XPUExpertsFp8, XPUExpertsMxfp8]
|
||||
|
||||
elif backend == Fp8MoeBackend.CPU:
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
|
||||
@@ -18,11 +18,13 @@ logger = init_logger(__name__)
|
||||
_SUPPORTED_BACKENDS = (
|
||||
Fp8MoeBackend.FLASHINFER_TRTLLM,
|
||||
Fp8MoeBackend.MARLIN,
|
||||
Fp8MoeBackend.XPU,
|
||||
)
|
||||
|
||||
_BACKEND_NAME_MAP: dict[str, Fp8MoeBackend] = {
|
||||
"flashinfer_trtllm": Fp8MoeBackend.FLASHINFER_TRTLLM,
|
||||
"marlin": Fp8MoeBackend.MARLIN,
|
||||
"xpu": Fp8MoeBackend.XPU,
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user