[MoE] Defer the TRTLLM-Gen top-k finalize on the modular path (#58635)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
24928a4d62
commit
378504a544
@@ -115,10 +115,11 @@ class LatentMoERunner(MoERunner):
|
||||
experts_cls = getattr(self._quant_method, "experts_cls", None)
|
||||
# The kernel instance is built after weight loading, while the tail
|
||||
# must register its CuTeDSL warmup units during runner construction.
|
||||
self.moe_config.defer_moe_finalize = (
|
||||
if (
|
||||
experts_cls is TrtLlmMxfp4ExpertsMonolithic
|
||||
or experts_cls is TrtLlmNvFp4ExpertsMonolithic
|
||||
) and self.moe_config.hidden_dim == self.moe_config.hidden_dim_unpadded
|
||||
) and self.moe_config.hidden_dim == self.moe_config.hidden_dim_unpadded:
|
||||
self.moe_config.defer_moe_finalize()
|
||||
from vllm.models.kimi_k3.nvidia.ops.latent_moe_tail import (
|
||||
KimiK3LatentMoETailOp,
|
||||
)
|
||||
@@ -136,20 +137,13 @@ class LatentMoERunner(MoERunner):
|
||||
rms_eps=norm.variance_epsilon,
|
||||
)
|
||||
self._k3_latent_moe_tail_op = op
|
||||
self.moe_config.defer_moe_finalize_max_num_tokens = (
|
||||
op.contract.max_num_tokens
|
||||
if self.moe_config.use_deferred_moe_finalize
|
||||
else -1
|
||||
)
|
||||
if self.moe_config.use_deferred_moe_finalize:
|
||||
self.moe_config.limit_deferred_moe_finalize(op.contract.max_num_tokens)
|
||||
logger.info_once(
|
||||
"K3 latent-MoE tail fusion with deferred top-k finalization "
|
||||
"is enabled for up to %d tokens.",
|
||||
op.contract.max_num_tokens,
|
||||
)
|
||||
else:
|
||||
self.moe_config.defer_moe_finalize = False
|
||||
self.moe_config.defer_moe_finalize_max_num_tokens = -1
|
||||
|
||||
def _get_zero_residual(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user