[MoE] Defer the TRTLLM-Gen top-k finalize on the modular path (#58635)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yongye Zhu
2026-09-25 15:38:29 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 24928a4d62
commit 378504a544
26 changed files with 652 additions and 117 deletions
@@ -115,10 +115,11 @@ class LatentMoERunner(MoERunner):
experts_cls = getattr(self._quant_method, "experts_cls", None)
# The kernel instance is built after weight loading, while the tail
# must register its CuTeDSL warmup units during runner construction.
self.moe_config.defer_moe_finalize = (
if (
experts_cls is TrtLlmMxfp4ExpertsMonolithic
or experts_cls is TrtLlmNvFp4ExpertsMonolithic
) and self.moe_config.hidden_dim == self.moe_config.hidden_dim_unpadded
) and self.moe_config.hidden_dim == self.moe_config.hidden_dim_unpadded:
self.moe_config.defer_moe_finalize()
from vllm.models.kimi_k3.nvidia.ops.latent_moe_tail import (
KimiK3LatentMoETailOp,
)
@@ -136,20 +137,13 @@ class LatentMoERunner(MoERunner):
rms_eps=norm.variance_epsilon,
)
self._k3_latent_moe_tail_op = op
self.moe_config.defer_moe_finalize_max_num_tokens = (
op.contract.max_num_tokens
if self.moe_config.use_deferred_moe_finalize
else -1
)
if self.moe_config.use_deferred_moe_finalize:
self.moe_config.limit_deferred_moe_finalize(op.contract.max_num_tokens)
logger.info_once(
"K3 latent-MoE tail fusion with deferred top-k finalization "
"is enabled for up to %d tokens.",
op.contract.max_num_tokens,
)
else:
self.moe_config.defer_moe_finalize = False
self.moe_config.defer_moe_finalize_max_num_tokens = -1
def _get_zero_residual(
self,