[Feature] Migrate moe sp support to non-torch compiled path for GLM5.2 (#47881)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Wentao Ye
2026-07-15 23:33:15 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent eb33ff34dd
commit 4238b011a7
4 changed files with 92 additions and 16 deletions
@@ -122,7 +122,9 @@ class TrtLlmFp8ExpertsBase:
return (
not moe_parallel_config.use_all2all_kernels
or moe_parallel_config.use_ag_rs_all2all_kernels
) and not moe_parallel_config.enable_eplb
) and not (
moe_parallel_config.enable_eplb or moe_parallel_config.is_sequence_parallel
)
class TrtLlmFp8ExpertsModular(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsModular):
+3 -1
View File
@@ -45,9 +45,10 @@ def _restore_full_token_layout_if_needed(
hidden_states: torch.Tensor,
residual: torch.Tensor,
num_tokens: int,
is_sequence_parallel: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Restore full token rows for the MTP proposer after SP MoE layers."""
if hidden_states.shape[0] == num_tokens:
if not is_sequence_parallel and hidden_states.shape[0] == num_tokens:
return hidden_states, residual
combined_states = torch.cat([hidden_states, residual], dim=-1)
@@ -142,6 +143,7 @@ class DeepSeekMultiTokenPredictorLayer(nn.Module):
hidden_states,
residual,
positions.shape[0],
is_sequence_parallel=self.mtp_block.use_sequence_parallel_moe,
)
hidden_states = residual + hidden_states # pre-final-norm (logits hidden)
# Recycle the post-final-norm hidden into the next draft step.
+82 -10
View File
@@ -7,7 +7,11 @@ from itertools import islice
import torch
from vllm.config import VllmConfig
from vllm.distributed import get_pp_group
from vllm.distributed import (
get_pp_group,
tensor_model_parallel_all_gather,
tensor_model_parallel_reduce_scatter,
)
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
@@ -32,6 +36,7 @@ from vllm.model_executor.models.utils import (
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
sequence_parallel_chunk,
)
from vllm.sequence import IntermediateTensors
@@ -39,6 +44,18 @@ from .attention import DeepseekV32Attention
from .fused_ops import fused_allreduce_rms_norm
def _all_gather_sp_states(
hidden_states: torch.Tensor,
residual: torch.Tensor,
num_tokens: int,
) -> tuple[torch.Tensor, torch.Tensor]:
# combine hidden_states and residual and all gather once
combined_states = torch.cat([hidden_states, residual], dim=-1)
combined_states = tensor_model_parallel_all_gather(combined_states, 0)[:num_tokens]
hidden_states, residual = combined_states.chunk(2, dim=-1)
return hidden_states, residual.contiguous()
class DeepseekV32DecoderLayer(torch.nn.Module):
def __init__(
self,
@@ -92,6 +109,12 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
prefix=f"{prefix}.mlp",
reduce_results=False,
)
self.use_sequence_parallel_moe = (
parallel_config.use_sequence_parallel_moe
and parallel_config.pipeline_parallel_size == 1
and isinstance(self.mlp, DeepseekV2MoE)
)
self.tp_size = parallel_config.tensor_parallel_size
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
@@ -104,25 +127,53 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
hidden_states: torch.Tensor,
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
full_num_tokens = positions.shape[0]
input_is_sequence_parallel = (
self.use_sequence_parallel_moe
and residual is not None
and hidden_states.shape[0] != full_num_tokens
)
if residual is None:
# First layer: hidden_states is the (already reduced) embedding.
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
elif input_is_sequence_parallel:
hidden_states, residual = self.input_layernorm(hidden_states, residual)
else:
# The previous layer's MLP/MoE output is left un-reduced; fuse its
# all-reduce into this input_layernorm.
hidden_states, residual = fused_allreduce_rms_norm(
hidden_states, residual, self.input_layernorm
)
# self_attn's o_proj runs reduce_results=False; fuse its all-reduce with
# the post-attention RMSNorm.
if input_is_sequence_parallel:
hidden_states = tensor_model_parallel_all_gather(hidden_states, 0)
hidden_states = hidden_states[:full_num_tokens]
# self_attn's o_proj runs reduce_results=False; reduce before RMSNorm.
hidden_states = self.self_attn(positions=positions, hidden_states=hidden_states)
hidden_states, residual = fused_allreduce_rms_norm(
hidden_states, residual, self.post_attention_layernorm
)
if self.use_sequence_parallel_moe:
# small trick using minus, eg. -17 % 8 = 7
sp_pad = (-hidden_states.shape[0]) % self.tp_size
# pad if not divisible by world size
hidden_states = torch.nn.functional.pad(hidden_states, (0, 0, 0, sp_pad))
hidden_states = tensor_model_parallel_reduce_scatter(hidden_states, 0)
if not input_is_sequence_parallel:
residual = sequence_parallel_chunk(residual)
hidden_states, residual = self.post_attention_layernorm(
hidden_states, residual
)
else:
hidden_states, residual = fused_allreduce_rms_norm(
hidden_states, residual, self.post_attention_layernorm
)
# MLP/MoE runs un-reduced; its all-reduce is fused into the next layer's
# input_layernorm (or the model's final norm).
hidden_states = self.mlp(hidden_states)
if self.use_sequence_parallel_moe:
hidden_states = self.mlp(hidden_states, already_sequence_parallel=True)
else:
hidden_states = self.mlp(hidden_states)
return hidden_states, residual
@@ -206,12 +257,26 @@ class DeepseekV32Model(torch.nn.Module):
residual = intermediate_tensors["residual"]
aux_hidden_states = []
full_num_tokens = positions.shape[0]
for idx, layer in enumerate(
islice(self.layers, self.start_layer, self.end_layer),
start=self.start_layer,
):
if (
hidden_states.shape[0] != full_num_tokens
and not layer.use_sequence_parallel_moe
):
hidden_states, residual = _all_gather_sp_states(
hidden_states, residual, full_num_tokens
)
if idx in self.aux_hidden_state_layers:
aux_hidden_states.append(hidden_states + residual)
aux_hidden_state = hidden_states + residual
if aux_hidden_state.shape[0] != full_num_tokens:
aux_hidden_state = tensor_model_parallel_all_gather(
aux_hidden_state, 0
)
aux_hidden_state = aux_hidden_state[:full_num_tokens]
aux_hidden_states.append(aux_hidden_state)
hidden_states, residual = layer(positions, hidden_states, residual)
if not get_pp_group().is_last_rank:
@@ -219,8 +284,15 @@ class DeepseekV32Model(torch.nn.Module):
{"hidden_states": hidden_states, "residual": residual}
)
# Last layer's MoE output is un-reduced; fuse its all-reduce into norm.
hidden_states, _ = fused_allreduce_rms_norm(hidden_states, residual, self.norm)
if hidden_states.shape[0] != full_num_tokens:
hidden_states, residual = _all_gather_sp_states(
hidden_states, residual, full_num_tokens
)
hidden_states, _ = self.norm(hidden_states, residual)
else:
hidden_states, _ = fused_allreduce_rms_norm(
hidden_states, residual, self.norm
)
if len(aux_hidden_states) > 0:
return hidden_states, aux_hidden_states
return hidden_states
+4 -4
View File
@@ -96,11 +96,11 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
hidden_states,
residual,
positions.shape[0],
is_sequence_parallel=self.mtp_block.use_sequence_parallel_moe,
)
# mtp_block's MoE output is left un-reduced (skip_final_all_reduce); the
# main model fuses that all-reduce into the next norm, but here the
# recycle hidden is consumed directly, so reduce it now.
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
if not self.mtp_block.use_sequence_parallel_moe:
# Without sequence parallelism, the MoE output is left un-reduced.
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
# Recycle the POST-final-norm hidden into the next draft step. The
# residual-add is fused into the final RMSNorm so it is computed
# exactly once, and the result is returned for both tuple positions: