[XPU] reudce host overhead of XPU MOE (#42915)
Signed-off-by: mayuyuace <qiming1.zhang@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
co-authored by
Kunshang Ji
parent
a5bbd81e2e
commit
54d153637b
@@ -23,7 +23,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_xpu():
|
||||
from vllm_xpu_kernels.fused_moe_interface import xpu_fused_moe
|
||||
from vllm_xpu_kernels.fused_moe_interface import XpuFusedMoe
|
||||
|
||||
|
||||
def prepare_fp8_moe_layer_for_xpu(
|
||||
@@ -50,6 +50,7 @@ class XPUExperts(mk.FusedMoEExpertsModular):
|
||||
self.is_fp8 = False
|
||||
self.is_mxfp4 = False
|
||||
self.is_mxfp8 = False
|
||||
self.fused_moe_impl: XpuFusedMoe | None = None
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
@@ -132,26 +133,30 @@ class XPUExperts(mk.FusedMoEExpertsModular):
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
):
|
||||
topk = topk_ids.size(-1)
|
||||
xpu_fused_moe(
|
||||
if self.fused_moe_impl is None:
|
||||
topk = topk_ids.size(-1)
|
||||
self.fused_moe_impl = XpuFusedMoe(
|
||||
w13=w1,
|
||||
w13_scales=self.w1_scale,
|
||||
w13_bias=self.w1_bias,
|
||||
w2=w2,
|
||||
w2_scales=self.w2_scale,
|
||||
w2_bias=self.w2_bias,
|
||||
n_experts_per_token=topk,
|
||||
activation=activation.value,
|
||||
num_experts=self.moe_config.num_local_experts,
|
||||
ep_rank=self.moe_config.ep_rank,
|
||||
ep_size=self.moe_config.ep_size,
|
||||
is_fp8=self.is_fp8,
|
||||
is_mxfp4=self.is_mxfp4,
|
||||
is_mxfp8=self.is_mxfp8,
|
||||
)
|
||||
assert self.fused_moe_impl is not None
|
||||
self.fused_moe_impl.apply(
|
||||
output=output,
|
||||
hidden_states=hidden_states,
|
||||
w13=w1,
|
||||
w13_scales=self.w1_scale,
|
||||
w13_bias=self.w1_bias,
|
||||
w2=w2,
|
||||
w2_scales=self.w2_scale,
|
||||
w2_bias=self.w2_bias,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
n_experts_per_token=topk,
|
||||
activation=activation.value,
|
||||
num_experts=self.moe_config.num_local_experts,
|
||||
ep_rank=self.moe_config.ep_rank,
|
||||
ep_size=self.moe_config.ep_size,
|
||||
output=output,
|
||||
is_fp8=self.is_fp8,
|
||||
is_mxfp4=self.is_mxfp4,
|
||||
is_mxfp8=self.is_mxfp8,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user