[Perf] Optimize DeepSeek V3.2 sequence parallelism (#51434)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
@@ -0,0 +1,148 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.models.deepseek_v32.nvidia import model as deepseek_v32_model
|
||||
from vllm.models.deepseek_v32.nvidia import mtp as deepseek_v32_mtp
|
||||
|
||||
|
||||
class _IdentityNorm(nn.Module):
|
||||
def __init__(self, hidden_size: int = 2) -> None:
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(hidden_size), requires_grad=False)
|
||||
self.variance_epsilon = 1e-5
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None = None,
|
||||
):
|
||||
if residual is None:
|
||||
return hidden_states
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
class _RecordingModule(nn.Module):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.num_tokens = 0
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor, **kwargs) -> torch.Tensor:
|
||||
self.num_tokens = hidden_states.shape[0]
|
||||
return hidden_states
|
||||
|
||||
|
||||
class _RecordingProjection(nn.Module):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(2, 4), requires_grad=False)
|
||||
self.num_tokens = 0
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
self.num_tokens = hidden_states.shape[0]
|
||||
return hidden_states[:, :2]
|
||||
|
||||
|
||||
class _SequenceParallelMTPBlock:
|
||||
use_sequence_parallel = True
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
*,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
):
|
||||
assert residual is None
|
||||
return hidden_states * 2, hidden_states * 3
|
||||
|
||||
|
||||
def _mock_sequence_parallel_collectives(monkeypatch, module):
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"sp_reduce_scatter",
|
||||
lambda tensor: tensor.chunk(2, dim=0)[0],
|
||||
raising=False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"sp_shard",
|
||||
lambda tensor: torch.nn.functional.pad(tensor, (0, 0, 0, 1))[:2],
|
||||
raising=False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"sp_all_gather",
|
||||
lambda tensor: torch.cat([tensor, tensor], dim=0),
|
||||
)
|
||||
|
||||
|
||||
def test_decoder_layer_keeps_dense_states_sequence_sharded(monkeypatch):
|
||||
layer = object.__new__(deepseek_v32_model.DeepseekV32DecoderLayer)
|
||||
nn.Module.__init__(layer)
|
||||
layer.use_sequence_parallel = True
|
||||
layer.input_layernorm = _IdentityNorm()
|
||||
layer.post_attention_layernorm = _IdentityNorm()
|
||||
layer.self_attn = _RecordingModule()
|
||||
layer.mlp = _RecordingModule()
|
||||
|
||||
_mock_sequence_parallel_collectives(monkeypatch, deepseek_v32_model)
|
||||
|
||||
positions = torch.arange(3)
|
||||
full_hidden_states = torch.arange(6, dtype=torch.float32).view(3, 2)
|
||||
hidden_states = deepseek_v32_model.sp_shard(full_hidden_states)
|
||||
hidden_states, residual = layer(positions, hidden_states, residual=None)
|
||||
|
||||
assert hidden_states.shape == residual.shape == (2, 2)
|
||||
assert layer.self_attn.num_tokens == 3
|
||||
assert layer.mlp.num_tokens == 2
|
||||
|
||||
hidden_states, residual = layer(positions, hidden_states, residual)
|
||||
|
||||
assert hidden_states.shape == residual.shape == (2, 2)
|
||||
assert layer.self_attn.num_tokens == 3
|
||||
assert layer.mlp.num_tokens == 2
|
||||
|
||||
|
||||
def test_mtp_projects_sequence_shard_and_restores_full_output(monkeypatch):
|
||||
layer = object.__new__(deepseek_v32_mtp.DeepseekV32MultiTokenPredictorLayer)
|
||||
nn.Module.__init__(layer)
|
||||
layer.enorm = _IdentityNorm()
|
||||
layer.hnorm = _IdentityNorm()
|
||||
layer.eh_proj = _RecordingProjection()
|
||||
layer._eh_plan = None
|
||||
object.__setattr__(layer, "mtp_block", _SequenceParallelMTPBlock())
|
||||
norm = Mock(
|
||||
side_effect=lambda hidden_states, residual: (hidden_states + residual, None)
|
||||
)
|
||||
object.__setattr__(layer, "shared_head", SimpleNamespace(norm=norm))
|
||||
|
||||
monkeypatch.setattr(
|
||||
deepseek_v32_mtp,
|
||||
"fused_eh_norm",
|
||||
lambda positions, inputs_embeds, previous_hidden_states, *args: torch.cat(
|
||||
[inputs_embeds, previous_hidden_states], dim=-1
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(deepseek_v32_mtp, "run_glm52_plan", lambda *args: None)
|
||||
_mock_sequence_parallel_collectives(monkeypatch, deepseek_v32_mtp)
|
||||
|
||||
inputs_embeds = torch.arange(6, dtype=torch.float32).view(3, 2)
|
||||
hidden_states, recycled_hidden_states = layer(
|
||||
input_ids=torch.zeros(3, dtype=torch.long),
|
||||
positions=torch.arange(3),
|
||||
previous_hidden_states=torch.zeros_like(inputs_embeds),
|
||||
inputs_embeds=inputs_embeds,
|
||||
)
|
||||
|
||||
sharded_states = torch.nn.functional.pad(inputs_embeds, (0, 0, 0, 1))[:2]
|
||||
expected = torch.cat([sharded_states * 5, sharded_states * 5])[:3]
|
||||
assert layer.eh_proj.num_tokens == 2
|
||||
torch.testing.assert_close(hidden_states, expected)
|
||||
torch.testing.assert_close(recycled_hidden_states, expected)
|
||||
norm.assert_called_once()
|
||||
@@ -6,12 +6,10 @@ from itertools import islice
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_pp_group,
|
||||
tensor_model_parallel_all_gather,
|
||||
tensor_model_parallel_reduce_scatter,
|
||||
)
|
||||
from vllm.distributed import get_pp_group
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
@@ -36,27 +34,20 @@ from vllm.model_executor.models.utils import (
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
sequence_parallel_chunk,
|
||||
)
|
||||
from vllm.models.common.ops.fused_allreduce_rms_norm import fused_allreduce_rms_norm
|
||||
from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_all_gather,
|
||||
sp_padding_mask,
|
||||
sp_reduce_scatter,
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.models.deepseek_v32.attention import DeepseekV32Attention
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .glm52_low_latency_gemm import enable_glm52_low_latency_gemm
|
||||
|
||||
|
||||
def _all_gather_sp_states(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
num_tokens: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# combine hidden_states and residual and all gather once
|
||||
combined_states = torch.cat([hidden_states, residual], dim=-1)
|
||||
combined_states = tensor_model_parallel_all_gather(combined_states, 0)[:num_tokens]
|
||||
hidden_states, residual = combined_states.chunk(2, dim=-1)
|
||||
return hidden_states, residual.contiguous()
|
||||
|
||||
|
||||
class DeepseekV32DecoderLayer(torch.nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -77,6 +68,10 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
|
||||
layer_idx = int(prefix.split(sep=".")[-1])
|
||||
self.layer_idx = layer_idx
|
||||
self.use_mha = False
|
||||
self.use_sequence_parallel = (
|
||||
parallel_config.use_sequence_parallel_moe
|
||||
and parallel_config.pipeline_parallel_size == 1
|
||||
)
|
||||
|
||||
self.self_attn = DeepseekV32Attention(
|
||||
vllm_config=vllm_config,
|
||||
@@ -90,9 +85,8 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
|
||||
and layer_idx >= config.first_k_dense_replace
|
||||
and layer_idx % moe_layer_freq == 0
|
||||
):
|
||||
# Defer the MoE cross-rank all-reduce; it is fused into the next
|
||||
# layer's input_layernorm (or the final norm) via
|
||||
# fused_allreduce_rms_norm.
|
||||
# Keep the MoE output un-reduced. Non-SP fuses its all-reduce into
|
||||
# the next layer norm; SP keeps the token shard local.
|
||||
self.mlp = DeepseekV2MoE(
|
||||
config=config,
|
||||
parallel_config=parallel_config,
|
||||
@@ -109,13 +103,8 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
reduce_results=False,
|
||||
is_sequence_parallel=self.use_sequence_parallel,
|
||||
)
|
||||
self.use_sequence_parallel_moe = (
|
||||
parallel_config.use_sequence_parallel_moe
|
||||
and parallel_config.pipeline_parallel_size == 1
|
||||
and isinstance(self.mlp, DeepseekV2MoE)
|
||||
)
|
||||
self.tp_size = parallel_config.tensor_parallel_size
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
@@ -129,17 +118,11 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
|
||||
residual: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
full_num_tokens = positions.shape[0]
|
||||
input_is_sequence_parallel = (
|
||||
self.use_sequence_parallel_moe
|
||||
and residual is not None
|
||||
and hidden_states.shape[0] != full_num_tokens
|
||||
)
|
||||
|
||||
if residual is None:
|
||||
# First layer: hidden_states is the (already reduced) embedding.
|
||||
residual = hidden_states
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
elif input_is_sequence_parallel:
|
||||
elif self.use_sequence_parallel:
|
||||
hidden_states, residual = self.input_layernorm(hidden_states, residual)
|
||||
else:
|
||||
# The previous layer's MLP/MoE output is left un-reduced; fuse its
|
||||
@@ -147,20 +130,13 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
|
||||
hidden_states, residual = fused_allreduce_rms_norm(
|
||||
hidden_states, residual, self.input_layernorm
|
||||
)
|
||||
if input_is_sequence_parallel:
|
||||
hidden_states = tensor_model_parallel_all_gather(hidden_states, 0)
|
||||
hidden_states = hidden_states[:full_num_tokens]
|
||||
if self.use_sequence_parallel:
|
||||
hidden_states = sp_all_gather(hidden_states)[:full_num_tokens]
|
||||
|
||||
# self_attn's o_proj runs reduce_results=False; reduce before RMSNorm.
|
||||
hidden_states = self.self_attn(positions=positions, hidden_states=hidden_states)
|
||||
if self.use_sequence_parallel_moe:
|
||||
# small trick using minus, eg. -17 % 8 = 7
|
||||
sp_pad = (-hidden_states.shape[0]) % self.tp_size
|
||||
# pad if not divisible by world size
|
||||
hidden_states = torch.nn.functional.pad(hidden_states, (0, 0, 0, sp_pad))
|
||||
hidden_states = tensor_model_parallel_reduce_scatter(hidden_states, 0)
|
||||
if not input_is_sequence_parallel:
|
||||
residual = sequence_parallel_chunk(residual)
|
||||
if self.use_sequence_parallel:
|
||||
hidden_states = sp_reduce_scatter(hidden_states)
|
||||
hidden_states, residual = self.post_attention_layernorm(
|
||||
hidden_states, residual
|
||||
)
|
||||
@@ -169,9 +145,7 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
|
||||
hidden_states, residual, self.post_attention_layernorm
|
||||
)
|
||||
|
||||
# MLP/MoE runs un-reduced; its all-reduce is fused into the next layer's
|
||||
# input_layernorm (or the model's final norm).
|
||||
if self.use_sequence_parallel_moe:
|
||||
if self.use_sequence_parallel and isinstance(self.mlp, DeepseekV2MoE):
|
||||
hidden_states = self.mlp(hidden_states, already_sequence_parallel=True)
|
||||
else:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
@@ -192,6 +166,11 @@ class DeepseekV32Model(torch.nn.Module):
|
||||
self.device = current_platform.device_type
|
||||
|
||||
self.vocab_size = config.vocab_size
|
||||
parallel_config = vllm_config.parallel_config
|
||||
self.use_sequence_parallel = (
|
||||
parallel_config.use_sequence_parallel_moe
|
||||
and parallel_config.pipeline_parallel_size == 1
|
||||
)
|
||||
# DSA is always sparse (has index_topk); allocate the shared top-k
|
||||
# buffer the indexer writes and the sparse MLA backend reads.
|
||||
self.is_v32 = True
|
||||
@@ -231,9 +210,7 @@ class DeepseekV32Model(torch.nn.Module):
|
||||
)
|
||||
|
||||
self.aux_hidden_state_layers = tuple[int, ...]()
|
||||
self.num_redundant_experts = (
|
||||
vllm_config.parallel_config.eplb_config.num_redundant_experts
|
||||
)
|
||||
self.num_redundant_experts = parallel_config.eplb_config.num_redundant_experts
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
@@ -257,39 +234,49 @@ class DeepseekV32Model(torch.nn.Module):
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
residual = intermediate_tensors["residual"]
|
||||
|
||||
aux_hidden_states = []
|
||||
full_num_tokens = positions.shape[0]
|
||||
if self.use_sequence_parallel:
|
||||
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
|
||||
forward_context = get_forward_context()
|
||||
forward_context.is_padding = sp_padding_mask(
|
||||
forward_context.is_padding, hidden_states
|
||||
)
|
||||
hidden_states = sp_shard(hidden_states)
|
||||
assert residual is None, "Currently, SP is not supported with PP"
|
||||
|
||||
aux_hidden_states = []
|
||||
for idx, layer in enumerate(
|
||||
islice(self.layers, self.start_layer, self.end_layer),
|
||||
start=self.start_layer,
|
||||
):
|
||||
if (
|
||||
hidden_states.shape[0] != full_num_tokens
|
||||
and not layer.use_sequence_parallel_moe
|
||||
):
|
||||
hidden_states, residual = _all_gather_sp_states(
|
||||
hidden_states, residual, full_num_tokens
|
||||
)
|
||||
if idx in self.aux_hidden_state_layers:
|
||||
aux_hidden_state = hidden_states + residual
|
||||
if aux_hidden_state.shape[0] != full_num_tokens:
|
||||
aux_hidden_state = tensor_model_parallel_all_gather(
|
||||
aux_hidden_state, 0
|
||||
)
|
||||
aux_hidden_state = aux_hidden_state[:full_num_tokens]
|
||||
aux_hidden_states.append(aux_hidden_state)
|
||||
aux_hidden_states.append(
|
||||
hidden_states if residual is None else hidden_states + residual
|
||||
)
|
||||
hidden_states, residual = layer(positions, hidden_states, residual)
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
assert not self.use_sequence_parallel, (
|
||||
"Currently, SP is not supported with PP"
|
||||
)
|
||||
return IntermediateTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
|
||||
if hidden_states.shape[0] != full_num_tokens:
|
||||
hidden_states, residual = _all_gather_sp_states(
|
||||
hidden_states, residual, full_num_tokens
|
||||
)
|
||||
if self.use_sequence_parallel:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
if aux_hidden_states:
|
||||
hidden_size = hidden_states.shape[-1]
|
||||
packed_hidden_states = torch.cat(
|
||||
[hidden_states, *aux_hidden_states], dim=-1
|
||||
)
|
||||
packed_hidden_states = sp_all_gather(packed_hidden_states)
|
||||
packed_hidden_states = packed_hidden_states[:full_num_tokens]
|
||||
hidden_states, *aux_hidden_states = packed_hidden_states.split(
|
||||
hidden_size, dim=-1
|
||||
)
|
||||
else:
|
||||
hidden_states = sp_all_gather(hidden_states)[:full_num_tokens]
|
||||
else:
|
||||
hidden_states, _ = fused_allreduce_rms_norm(
|
||||
hidden_states, residual, self.norm
|
||||
|
||||
@@ -6,12 +6,11 @@ from collections.abc import Callable, Iterable
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import (
|
||||
tensor_model_parallel_all_gather,
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from vllm.distributed import tensor_model_parallel_all_reduce
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
@@ -38,6 +37,11 @@ from vllm.model_executor.models.utils import (
|
||||
get_pp_missing_layer_names,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_all_gather,
|
||||
sp_padding_mask,
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.models.deepseek_v32.common.kernels import fused_eh_norm
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -101,13 +105,20 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
|
||||
self.hnorm.weight,
|
||||
self.enorm.variance_epsilon,
|
||||
)
|
||||
is_sequence_parallel = self.mtp_block.use_sequence_parallel
|
||||
if is_sequence_parallel:
|
||||
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
|
||||
forward_context = get_forward_context()
|
||||
forward_context.is_padding = sp_padding_mask(
|
||||
forward_context.is_padding, eh_input
|
||||
)
|
||||
eh_input = sp_shard(eh_input)
|
||||
hidden_states = run_glm52_plan(self._eh_plan, eh_input, self.eh_proj.weight)
|
||||
if hidden_states is None:
|
||||
hidden_states = self.eh_proj(eh_input)
|
||||
hidden_states, residual = self.mtp_block(
|
||||
positions=positions, hidden_states=hidden_states, residual=None
|
||||
)
|
||||
is_sequence_parallel = self.mtp_block.use_sequence_parallel_moe
|
||||
if not is_sequence_parallel:
|
||||
# Without sequence parallelism, the MoE output is left un-reduced.
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
@@ -123,8 +134,7 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
|
||||
# DeepSeekMTPModel architecture).
|
||||
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
|
||||
if is_sequence_parallel:
|
||||
hidden_states = tensor_model_parallel_all_gather(hidden_states, 0)
|
||||
hidden_states = hidden_states[: positions.shape[0]]
|
||||
hidden_states = sp_all_gather(hidden_states)[: positions.shape[0]]
|
||||
return hidden_states, hidden_states
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user