[Perf] Update TRTLLM supported MoE routing methods (#39141)
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Signed-off-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: root <root@bia0030.bia.clusters.nvidia.com> Co-authored-by: root <root@bia0036.bia.clusters.nvidia.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
root
root
parent
6232fb4b66
commit
c8bbe05189
@@ -50,7 +50,6 @@ from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
GateLinear,
|
||||
RoutingMethodType,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import LayerNorm, RMSNorm
|
||||
@@ -338,17 +337,6 @@ class DeepseekV2MoE(nn.Module):
|
||||
else None,
|
||||
)
|
||||
|
||||
# NOTE(rob): this is a hack until we finish off the PR for
|
||||
# merging TRTLLM kernels into the MK framework. Then we can
|
||||
# query the MonolithicMK for the expected router logits.
|
||||
# NOTE(dbari): Use BF16 if routing is not Deepseek, e.g. Mistral Large 3
|
||||
self.gate.set_out_dtype(
|
||||
torch.float32
|
||||
if self.experts.quant_method.is_monolithic
|
||||
and self.experts.routing_method_type == RoutingMethodType.DeepSeekV3
|
||||
else torch.bfloat16
|
||||
)
|
||||
|
||||
# Pre-cast the bias to match the gate output dtype so the
|
||||
# conversion is not repeated on every forward pass. All
|
||||
# downstream references (FusedMoE, router) share the same
|
||||
|
||||
Reference in New Issue
Block a user