Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -825,8 +825,13 @@ class MoERunner(MoERunnerInterface):
|
||||
| tuple[torch.Tensor | None, torch.Tensor | UnfinalizedMoEOutput]
|
||||
):
|
||||
if self.do_naive_dispatch_combine:
|
||||
if isinstance(hidden_states, UnfinalizedMoEOutput):
|
||||
raise RuntimeError(
|
||||
"Naive expert-parallel combine cannot consume a deferred "
|
||||
"MoE output."
|
||||
)
|
||||
hidden_states = get_ep_group().combine(
|
||||
cast(torch.Tensor, hidden_states),
|
||||
hidden_states,
|
||||
self.moe_config.is_sequence_parallel,
|
||||
)
|
||||
|
||||
@@ -834,9 +839,11 @@ class MoERunner(MoERunnerInterface):
|
||||
self.moe_config.pcp_size > 1
|
||||
and not self.moe_config.moe_parallel_config.use_all2all_kernels
|
||||
):
|
||||
hidden_states = get_pcp_group().reduce_scatter(
|
||||
cast(torch.Tensor, hidden_states), dim=0
|
||||
)
|
||||
if isinstance(hidden_states, UnfinalizedMoEOutput):
|
||||
raise RuntimeError(
|
||||
"PCP reduce-scatter cannot consume a deferred MoE output."
|
||||
)
|
||||
hidden_states = get_pcp_group().reduce_scatter(hidden_states, dim=0)
|
||||
|
||||
if self.shared_experts is not None:
|
||||
assert shared_output is not None
|
||||
|
||||
@@ -17,6 +17,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
|
||||
FusedMoEMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
UnquantizedMoeBackend,
|
||||
convert_to_unquantized_kernel_format,
|
||||
@@ -299,7 +300,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply_monolithic(
|
||||
|
||||
+2
-1
@@ -14,6 +14,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
|
||||
NvFp4MoeBackend,
|
||||
convert_to_nvfp4_moe_kernel_format,
|
||||
@@ -278,7 +279,7 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply_monolithic(
|
||||
|
||||
@@ -24,6 +24,7 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
RoutedExperts,
|
||||
SharedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
Fp8MoeBackend,
|
||||
convert_to_fp8_moe_kernel_format,
|
||||
@@ -1624,7 +1625,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply_monolithic(
|
||||
|
||||
@@ -10,6 +10,7 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
RoutedExperts,
|
||||
SharedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.model_loader.reload.layerwise import (
|
||||
initialize_online_processing,
|
||||
)
|
||||
@@ -139,7 +140,7 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply_monolithic(
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
mxfp4_w4a16_moe_quant_config,
|
||||
ocp_mx_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
convert_to_fp8_moe_kernel_format,
|
||||
make_fp8_moe_kernel,
|
||||
@@ -1444,7 +1445,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply_monolithic(
|
||||
|
||||
Reference in New Issue
Block a user