[GLM5.2 Perf] Replace MOE all-reduce with reduce-scatter, 3.1%~3.2 E2E Throughput improvement (#46635)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -42,6 +42,7 @@ from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
tensor_model_parallel_all_gather,
|
||||
tensor_model_parallel_reduce_scatter,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
@@ -128,6 +129,7 @@ class DeepseekAttention(nn.Module):
|
||||
max_position_embeddings: int = 8192,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
reduce_results: bool = True,
|
||||
prefix: str = "",
|
||||
**kwargs,
|
||||
) -> None:
|
||||
@@ -166,6 +168,7 @@ class DeepseekAttention(nn.Module):
|
||||
self.total_num_heads * self.head_dim,
|
||||
hidden_size,
|
||||
bias=False,
|
||||
reduce_results=reduce_results,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
|
||||
@@ -372,15 +375,17 @@ class DeepseekV2MoE(nn.Module):
|
||||
self.gate.e_score_correction_bias.data.to(self.gate.out_dtype)
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
already_sequence_parallel: bool = False,
|
||||
) -> torch.Tensor:
|
||||
num_tokens, hidden_dim = hidden_states.shape
|
||||
hidden_states = hidden_states.view(-1, hidden_dim)
|
||||
|
||||
# Chunk the hidden states so they aren't replicated across TP ranks.
|
||||
# This avoids duplicate computation in self.experts.
|
||||
# TODO: We can replace the all_reduce at the end of attn with a
|
||||
# reduce_scatter instead of chunking here.
|
||||
if self.is_sequence_parallel:
|
||||
if self.is_sequence_parallel and not already_sequence_parallel:
|
||||
hidden_states = sequence_parallel_chunk(hidden_states)
|
||||
|
||||
if self.experts.is_internal_router:
|
||||
@@ -393,7 +398,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
hidden_states=hidden_states, router_logits=router_logits
|
||||
)
|
||||
|
||||
if self.is_sequence_parallel:
|
||||
if self.is_sequence_parallel and not already_sequence_parallel:
|
||||
final_hidden_states = tensor_model_parallel_all_gather(
|
||||
final_hidden_states, 0
|
||||
)
|
||||
@@ -436,6 +441,7 @@ class DeepseekV2Attention(nn.Module):
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
reduce_results: bool = True,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
@@ -502,6 +508,7 @@ class DeepseekV2Attention(nn.Module):
|
||||
self.num_heads * self.v_head_dim,
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
reduce_results=reduce_results,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
@@ -950,6 +957,7 @@ class DeepseekV2MLAAttention(nn.Module):
|
||||
prefix: str = "",
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
input_size: int | None = None,
|
||||
reduce_results: bool = True,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
@@ -1018,6 +1026,7 @@ class DeepseekV2MLAAttention(nn.Module):
|
||||
self.num_heads * self.v_head_dim,
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
reduce_results=reduce_results,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
@@ -1184,6 +1193,18 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
attn_cls = DeepseekV2MLAAttention
|
||||
else:
|
||||
attn_cls = DeepseekV2Attention
|
||||
is_moe_layer = (
|
||||
config.n_routed_experts is not None
|
||||
and layer_idx >= config.first_k_dense_replace
|
||||
and layer_idx % moe_layer_freq == 0
|
||||
)
|
||||
# TODO(wentao): enable SP MoE with PP after the PP boundary logic can safely
|
||||
# send/receive sequence-parallel hidden_states across stages.
|
||||
self.use_sequence_parallel_moe = (
|
||||
parallel_config.use_sequence_parallel_moe
|
||||
and parallel_config.pipeline_parallel_size == 1
|
||||
and is_moe_layer
|
||||
)
|
||||
self.self_attn = attn_cls(
|
||||
vllm_config=vllm_config,
|
||||
config=config,
|
||||
@@ -1199,13 +1220,10 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
reduce_results=not self.use_sequence_parallel_moe,
|
||||
)
|
||||
|
||||
if (
|
||||
config.n_routed_experts is not None
|
||||
and layer_idx >= config.first_k_dense_replace
|
||||
and layer_idx % moe_layer_freq == 0
|
||||
):
|
||||
if is_moe_layer:
|
||||
self.mlp = DeepseekV2MoE(
|
||||
config=config,
|
||||
parallel_config=parallel_config,
|
||||
@@ -1233,6 +1251,13 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
residual: torch.Tensor | None,
|
||||
llama_4_scaling: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
full_num_tokens = positions.shape[0]
|
||||
input_is_sequence_parallel = (
|
||||
self.use_sequence_parallel_moe
|
||||
and residual is not None
|
||||
and hidden_states.shape[0] != full_num_tokens
|
||||
)
|
||||
|
||||
# Self Attention
|
||||
if residual is None:
|
||||
residual = hidden_states
|
||||
@@ -1240,6 +1265,10 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
else:
|
||||
hidden_states, residual = self.input_layernorm(hidden_states, residual)
|
||||
|
||||
if input_is_sequence_parallel:
|
||||
hidden_states = tensor_model_parallel_all_gather(hidden_states, 0)
|
||||
hidden_states = hidden_states[:full_num_tokens]
|
||||
|
||||
attn_kwargs = {
|
||||
"positions": positions,
|
||||
"hidden_states": hidden_states,
|
||||
@@ -1261,9 +1290,29 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
# first layer.
|
||||
residual *= 1.0 / self.routed_scaling_factor
|
||||
|
||||
if self.use_sequence_parallel_moe:
|
||||
sp_remainder = (
|
||||
hidden_states.shape[0] % get_tensor_model_parallel_world_size()
|
||||
)
|
||||
# pad if not divisible by world size
|
||||
if sp_remainder:
|
||||
sp_pad = get_tensor_model_parallel_world_size() - sp_remainder
|
||||
hidden_states = torch.nn.functional.pad(
|
||||
hidden_states, (0, 0, 0, sp_pad)
|
||||
)
|
||||
hidden_states = tensor_model_parallel_reduce_scatter(hidden_states, 0)
|
||||
if not input_is_sequence_parallel:
|
||||
residual = sequence_parallel_chunk(residual)
|
||||
|
||||
# Fully Connected
|
||||
hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
if self.use_sequence_parallel_moe:
|
||||
hidden_states = self.mlp(
|
||||
hidden_states,
|
||||
already_sequence_parallel=True,
|
||||
)
|
||||
else:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
|
||||
if isinstance(self.mlp, DeepseekV2MLP) and hidden_states.dtype == torch.float16:
|
||||
# Fix FP16 overflow
|
||||
@@ -1385,8 +1434,25 @@ class DeepseekV2Model(nn.Module):
|
||||
islice(self.layers, self.start_layer, self.end_layer),
|
||||
start=self.start_layer,
|
||||
):
|
||||
# all gather if we need to use the whole states
|
||||
if (
|
||||
hidden_states.shape[0] != positions.shape[0]
|
||||
and not layer.use_sequence_parallel_moe
|
||||
):
|
||||
combined_states = torch.cat([hidden_states, residual], dim=-1)
|
||||
combined_states = tensor_model_parallel_all_gather(combined_states, 0)
|
||||
combined_states = combined_states[: positions.shape[0]]
|
||||
hidden_states, residual = combined_states.split(
|
||||
[self.hidden_size, self.hidden_size], dim=-1
|
||||
)
|
||||
if idx in self.aux_hidden_state_layers:
|
||||
aux_hidden_states.append(hidden_states + residual)
|
||||
aux_hidden_state = hidden_states + residual
|
||||
if aux_hidden_state.shape[0] != positions.shape[0]:
|
||||
aux_hidden_state = tensor_model_parallel_all_gather(
|
||||
aux_hidden_state, 0
|
||||
)
|
||||
aux_hidden_state = aux_hidden_state[: positions.shape[0]]
|
||||
aux_hidden_states.append(aux_hidden_state)
|
||||
hidden_states, residual = layer(
|
||||
positions, hidden_states, residual, llama_4_scaling
|
||||
)
|
||||
@@ -1396,6 +1462,14 @@ class DeepseekV2Model(nn.Module):
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
|
||||
if hidden_states.shape[0] != positions.shape[0]:
|
||||
combined_states = torch.cat([hidden_states, residual], dim=-1)
|
||||
combined_states = tensor_model_parallel_all_gather(combined_states, 0)
|
||||
combined_states = combined_states[: positions.shape[0]]
|
||||
hidden_states, residual = combined_states.split(
|
||||
[self.hidden_size, self.hidden_size], dim=-1
|
||||
)
|
||||
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
if len(aux_hidden_states) > 0:
|
||||
return hidden_states, aux_hidden_states
|
||||
|
||||
Reference in New Issue
Block a user