[MoE] Defer the TRTLLM-Gen top-k finalize on the modular path (#58635)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
24928a4d62
commit
378504a544
@@ -23,6 +23,7 @@ from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
|
||||
FusedMoEMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
B12X_BACKENDS,
|
||||
Mxfp4MoeBackend,
|
||||
@@ -228,7 +229,7 @@ class INCMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
x,
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
|
||||
FusedMoEMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
convert_to_fp8_moe_kernel_format,
|
||||
make_fp8_moe_kernel,
|
||||
@@ -179,7 +180,7 @@ class INCMxfp8MoEMethod(FusedMoEMethodBase):
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
x,
|
||||
|
||||
Reference in New Issue
Block a user