[Feature] Migrate moe sp support to non-torch compiled path for GLM5.2 (#47881)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
eb33ff34dd
commit
4238b011a7
@@ -122,7 +122,9 @@ class TrtLlmFp8ExpertsBase:
|
||||
return (
|
||||
not moe_parallel_config.use_all2all_kernels
|
||||
or moe_parallel_config.use_ag_rs_all2all_kernels
|
||||
) and not moe_parallel_config.enable_eplb
|
||||
) and not (
|
||||
moe_parallel_config.enable_eplb or moe_parallel_config.is_sequence_parallel
|
||||
)
|
||||
|
||||
|
||||
class TrtLlmFp8ExpertsModular(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsModular):
|
||||
|
||||
@@ -45,9 +45,10 @@ def _restore_full_token_layout_if_needed(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
num_tokens: int,
|
||||
is_sequence_parallel: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Restore full token rows for the MTP proposer after SP MoE layers."""
|
||||
if hidden_states.shape[0] == num_tokens:
|
||||
if not is_sequence_parallel and hidden_states.shape[0] == num_tokens:
|
||||
return hidden_states, residual
|
||||
|
||||
combined_states = torch.cat([hidden_states, residual], dim=-1)
|
||||
@@ -142,6 +143,7 @@ class DeepSeekMultiTokenPredictorLayer(nn.Module):
|
||||
hidden_states,
|
||||
residual,
|
||||
positions.shape[0],
|
||||
is_sequence_parallel=self.mtp_block.use_sequence_parallel_moe,
|
||||
)
|
||||
hidden_states = residual + hidden_states # pre-final-norm (logits hidden)
|
||||
# Recycle the post-final-norm hidden into the next draft step.
|
||||
|
||||
@@ -7,7 +7,11 @@ from itertools import islice
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_pp_group
|
||||
from vllm.distributed import (
|
||||
get_pp_group,
|
||||
tensor_model_parallel_all_gather,
|
||||
tensor_model_parallel_reduce_scatter,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
@@ -32,6 +36,7 @@ from vllm.model_executor.models.utils import (
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
sequence_parallel_chunk,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
@@ -39,6 +44,18 @@ from .attention import DeepseekV32Attention
|
||||
from .fused_ops import fused_allreduce_rms_norm
|
||||
|
||||
|
||||
def _all_gather_sp_states(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
num_tokens: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# combine hidden_states and residual and all gather once
|
||||
combined_states = torch.cat([hidden_states, residual], dim=-1)
|
||||
combined_states = tensor_model_parallel_all_gather(combined_states, 0)[:num_tokens]
|
||||
hidden_states, residual = combined_states.chunk(2, dim=-1)
|
||||
return hidden_states, residual.contiguous()
|
||||
|
||||
|
||||
class DeepseekV32DecoderLayer(torch.nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -92,6 +109,12 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
|
||||
prefix=f"{prefix}.mlp",
|
||||
reduce_results=False,
|
||||
)
|
||||
self.use_sequence_parallel_moe = (
|
||||
parallel_config.use_sequence_parallel_moe
|
||||
and parallel_config.pipeline_parallel_size == 1
|
||||
and isinstance(self.mlp, DeepseekV2MoE)
|
||||
)
|
||||
self.tp_size = parallel_config.tensor_parallel_size
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
@@ -104,25 +127,53 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
full_num_tokens = positions.shape[0]
|
||||
input_is_sequence_parallel = (
|
||||
self.use_sequence_parallel_moe
|
||||
and residual is not None
|
||||
and hidden_states.shape[0] != full_num_tokens
|
||||
)
|
||||
|
||||
if residual is None:
|
||||
# First layer: hidden_states is the (already reduced) embedding.
|
||||
residual = hidden_states
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
elif input_is_sequence_parallel:
|
||||
hidden_states, residual = self.input_layernorm(hidden_states, residual)
|
||||
else:
|
||||
# The previous layer's MLP/MoE output is left un-reduced; fuse its
|
||||
# all-reduce into this input_layernorm.
|
||||
hidden_states, residual = fused_allreduce_rms_norm(
|
||||
hidden_states, residual, self.input_layernorm
|
||||
)
|
||||
# self_attn's o_proj runs reduce_results=False; fuse its all-reduce with
|
||||
# the post-attention RMSNorm.
|
||||
if input_is_sequence_parallel:
|
||||
hidden_states = tensor_model_parallel_all_gather(hidden_states, 0)
|
||||
hidden_states = hidden_states[:full_num_tokens]
|
||||
|
||||
# self_attn's o_proj runs reduce_results=False; reduce before RMSNorm.
|
||||
hidden_states = self.self_attn(positions=positions, hidden_states=hidden_states)
|
||||
hidden_states, residual = fused_allreduce_rms_norm(
|
||||
hidden_states, residual, self.post_attention_layernorm
|
||||
)
|
||||
if self.use_sequence_parallel_moe:
|
||||
# small trick using minus, eg. -17 % 8 = 7
|
||||
sp_pad = (-hidden_states.shape[0]) % self.tp_size
|
||||
# pad if not divisible by world size
|
||||
hidden_states = torch.nn.functional.pad(hidden_states, (0, 0, 0, sp_pad))
|
||||
hidden_states = tensor_model_parallel_reduce_scatter(hidden_states, 0)
|
||||
if not input_is_sequence_parallel:
|
||||
residual = sequence_parallel_chunk(residual)
|
||||
hidden_states, residual = self.post_attention_layernorm(
|
||||
hidden_states, residual
|
||||
)
|
||||
else:
|
||||
hidden_states, residual = fused_allreduce_rms_norm(
|
||||
hidden_states, residual, self.post_attention_layernorm
|
||||
)
|
||||
|
||||
# MLP/MoE runs un-reduced; its all-reduce is fused into the next layer's
|
||||
# input_layernorm (or the model's final norm).
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
if self.use_sequence_parallel_moe:
|
||||
hidden_states = self.mlp(hidden_states, already_sequence_parallel=True)
|
||||
else:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
@@ -206,12 +257,26 @@ class DeepseekV32Model(torch.nn.Module):
|
||||
residual = intermediate_tensors["residual"]
|
||||
|
||||
aux_hidden_states = []
|
||||
full_num_tokens = positions.shape[0]
|
||||
for idx, layer in enumerate(
|
||||
islice(self.layers, self.start_layer, self.end_layer),
|
||||
start=self.start_layer,
|
||||
):
|
||||
if (
|
||||
hidden_states.shape[0] != full_num_tokens
|
||||
and not layer.use_sequence_parallel_moe
|
||||
):
|
||||
hidden_states, residual = _all_gather_sp_states(
|
||||
hidden_states, residual, full_num_tokens
|
||||
)
|
||||
if idx in self.aux_hidden_state_layers:
|
||||
aux_hidden_states.append(hidden_states + residual)
|
||||
aux_hidden_state = hidden_states + residual
|
||||
if aux_hidden_state.shape[0] != full_num_tokens:
|
||||
aux_hidden_state = tensor_model_parallel_all_gather(
|
||||
aux_hidden_state, 0
|
||||
)
|
||||
aux_hidden_state = aux_hidden_state[:full_num_tokens]
|
||||
aux_hidden_states.append(aux_hidden_state)
|
||||
hidden_states, residual = layer(positions, hidden_states, residual)
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
@@ -219,8 +284,15 @@ class DeepseekV32Model(torch.nn.Module):
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
|
||||
# Last layer's MoE output is un-reduced; fuse its all-reduce into norm.
|
||||
hidden_states, _ = fused_allreduce_rms_norm(hidden_states, residual, self.norm)
|
||||
if hidden_states.shape[0] != full_num_tokens:
|
||||
hidden_states, residual = _all_gather_sp_states(
|
||||
hidden_states, residual, full_num_tokens
|
||||
)
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
else:
|
||||
hidden_states, _ = fused_allreduce_rms_norm(
|
||||
hidden_states, residual, self.norm
|
||||
)
|
||||
if len(aux_hidden_states) > 0:
|
||||
return hidden_states, aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
@@ -96,11 +96,11 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
|
||||
hidden_states,
|
||||
residual,
|
||||
positions.shape[0],
|
||||
is_sequence_parallel=self.mtp_block.use_sequence_parallel_moe,
|
||||
)
|
||||
# mtp_block's MoE output is left un-reduced (skip_final_all_reduce); the
|
||||
# main model fuses that all-reduce into the next norm, but here the
|
||||
# recycle hidden is consumed directly, so reduce it now.
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
if not self.mtp_block.use_sequence_parallel_moe:
|
||||
# Without sequence parallelism, the MoE output is left un-reduced.
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
# Recycle the POST-final-norm hidden into the next draft step. The
|
||||
# residual-add is fused into the final RMSNorm so it is computed
|
||||
# exactly once, and the result is returned for both tuple positions:
|
||||
|
||||
Reference in New Issue
Block a user