diff --git a/tests/models/deepseek_v32/test_sequence_parallel.py b/tests/models/deepseek_v32/test_sequence_parallel.py new file mode 100644 index 0000000000..c0378dd5e3 --- /dev/null +++ b/tests/models/deepseek_v32/test_sequence_parallel.py @@ -0,0 +1,148 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace +from unittest.mock import Mock + +import torch +from torch import nn + +from vllm.models.deepseek_v32.nvidia import model as deepseek_v32_model +from vllm.models.deepseek_v32.nvidia import mtp as deepseek_v32_mtp + + +class _IdentityNorm(nn.Module): + def __init__(self, hidden_size: int = 2) -> None: + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size), requires_grad=False) + self.variance_epsilon = 1e-5 + + def forward( + self, + hidden_states: torch.Tensor, + residual: torch.Tensor | None = None, + ): + if residual is None: + return hidden_states + return hidden_states, residual + + +class _RecordingModule(nn.Module): + def __init__(self) -> None: + super().__init__() + self.num_tokens = 0 + + def forward(self, hidden_states: torch.Tensor, **kwargs) -> torch.Tensor: + self.num_tokens = hidden_states.shape[0] + return hidden_states + + +class _RecordingProjection(nn.Module): + def __init__(self) -> None: + super().__init__() + self.weight = nn.Parameter(torch.ones(2, 4), requires_grad=False) + self.num_tokens = 0 + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + self.num_tokens = hidden_states.shape[0] + return hidden_states[:, :2] + + +class _SequenceParallelMTPBlock: + use_sequence_parallel = True + + def __call__( + self, + *, + positions: torch.Tensor, + hidden_states: torch.Tensor, + residual: torch.Tensor | None, + ): + assert residual is None + return hidden_states * 2, hidden_states * 3 + + +def _mock_sequence_parallel_collectives(monkeypatch, module): + monkeypatch.setattr( + module, + "sp_reduce_scatter", + lambda tensor: tensor.chunk(2, dim=0)[0], + raising=False, + ) + monkeypatch.setattr( + module, + "sp_shard", + lambda tensor: torch.nn.functional.pad(tensor, (0, 0, 0, 1))[:2], + raising=False, + ) + monkeypatch.setattr( + module, + "sp_all_gather", + lambda tensor: torch.cat([tensor, tensor], dim=0), + ) + + +def test_decoder_layer_keeps_dense_states_sequence_sharded(monkeypatch): + layer = object.__new__(deepseek_v32_model.DeepseekV32DecoderLayer) + nn.Module.__init__(layer) + layer.use_sequence_parallel = True + layer.input_layernorm = _IdentityNorm() + layer.post_attention_layernorm = _IdentityNorm() + layer.self_attn = _RecordingModule() + layer.mlp = _RecordingModule() + + _mock_sequence_parallel_collectives(monkeypatch, deepseek_v32_model) + + positions = torch.arange(3) + full_hidden_states = torch.arange(6, dtype=torch.float32).view(3, 2) + hidden_states = deepseek_v32_model.sp_shard(full_hidden_states) + hidden_states, residual = layer(positions, hidden_states, residual=None) + + assert hidden_states.shape == residual.shape == (2, 2) + assert layer.self_attn.num_tokens == 3 + assert layer.mlp.num_tokens == 2 + + hidden_states, residual = layer(positions, hidden_states, residual) + + assert hidden_states.shape == residual.shape == (2, 2) + assert layer.self_attn.num_tokens == 3 + assert layer.mlp.num_tokens == 2 + + +def test_mtp_projects_sequence_shard_and_restores_full_output(monkeypatch): + layer = object.__new__(deepseek_v32_mtp.DeepseekV32MultiTokenPredictorLayer) + nn.Module.__init__(layer) + layer.enorm = _IdentityNorm() + layer.hnorm = _IdentityNorm() + layer.eh_proj = _RecordingProjection() + layer._eh_plan = None + object.__setattr__(layer, "mtp_block", _SequenceParallelMTPBlock()) + norm = Mock( + side_effect=lambda hidden_states, residual: (hidden_states + residual, None) + ) + object.__setattr__(layer, "shared_head", SimpleNamespace(norm=norm)) + + monkeypatch.setattr( + deepseek_v32_mtp, + "fused_eh_norm", + lambda positions, inputs_embeds, previous_hidden_states, *args: torch.cat( + [inputs_embeds, previous_hidden_states], dim=-1 + ), + ) + monkeypatch.setattr(deepseek_v32_mtp, "run_glm52_plan", lambda *args: None) + _mock_sequence_parallel_collectives(monkeypatch, deepseek_v32_mtp) + + inputs_embeds = torch.arange(6, dtype=torch.float32).view(3, 2) + hidden_states, recycled_hidden_states = layer( + input_ids=torch.zeros(3, dtype=torch.long), + positions=torch.arange(3), + previous_hidden_states=torch.zeros_like(inputs_embeds), + inputs_embeds=inputs_embeds, + ) + + sharded_states = torch.nn.functional.pad(inputs_embeds, (0, 0, 0, 1))[:2] + expected = torch.cat([sharded_states * 5, sharded_states * 5])[:3] + assert layer.eh_proj.num_tokens == 2 + torch.testing.assert_close(hidden_states, expected) + torch.testing.assert_close(recycled_hidden_states, expected) + norm.assert_called_once() diff --git a/vllm/models/deepseek_v32/nvidia/model.py b/vllm/models/deepseek_v32/nvidia/model.py index f678e1a62d..0173d1ba6c 100644 --- a/vllm/models/deepseek_v32/nvidia/model.py +++ b/vllm/models/deepseek_v32/nvidia/model.py @@ -6,12 +6,10 @@ from itertools import islice import torch +import vllm.envs as envs from vllm.config import VllmConfig -from vllm.distributed import ( - get_pp_group, - tensor_model_parallel_all_gather, - tensor_model_parallel_reduce_scatter, -) +from vllm.distributed import get_pp_group +from vllm.forward_context import get_forward_context, is_forward_context_available from vllm.model_executor.layers.fused_moe import ( fused_moe_make_expert_params_mapping, ) @@ -36,27 +34,20 @@ from vllm.model_executor.models.utils import ( is_pp_missing_parameter, make_empty_intermediate_tensors_factory, make_layers, - sequence_parallel_chunk, ) from vllm.models.common.ops.fused_allreduce_rms_norm import fused_allreduce_rms_norm +from vllm.models.common.ops.sequence_parallel import ( + sp_all_gather, + sp_padding_mask, + sp_reduce_scatter, + sp_shard, +) from vllm.models.deepseek_v32.attention import DeepseekV32Attention from vllm.sequence import IntermediateTensors from .glm52_low_latency_gemm import enable_glm52_low_latency_gemm -def _all_gather_sp_states( - hidden_states: torch.Tensor, - residual: torch.Tensor, - num_tokens: int, -) -> tuple[torch.Tensor, torch.Tensor]: - # combine hidden_states and residual and all gather once - combined_states = torch.cat([hidden_states, residual], dim=-1) - combined_states = tensor_model_parallel_all_gather(combined_states, 0)[:num_tokens] - hidden_states, residual = combined_states.chunk(2, dim=-1) - return hidden_states, residual.contiguous() - - class DeepseekV32DecoderLayer(torch.nn.Module): def __init__( self, @@ -77,6 +68,10 @@ class DeepseekV32DecoderLayer(torch.nn.Module): layer_idx = int(prefix.split(sep=".")[-1]) self.layer_idx = layer_idx self.use_mha = False + self.use_sequence_parallel = ( + parallel_config.use_sequence_parallel_moe + and parallel_config.pipeline_parallel_size == 1 + ) self.self_attn = DeepseekV32Attention( vllm_config=vllm_config, @@ -90,9 +85,8 @@ class DeepseekV32DecoderLayer(torch.nn.Module): and layer_idx >= config.first_k_dense_replace and layer_idx % moe_layer_freq == 0 ): - # Defer the MoE cross-rank all-reduce; it is fused into the next - # layer's input_layernorm (or the final norm) via - # fused_allreduce_rms_norm. + # Keep the MoE output un-reduced. Non-SP fuses its all-reduce into + # the next layer norm; SP keeps the token shard local. self.mlp = DeepseekV2MoE( config=config, parallel_config=parallel_config, @@ -109,13 +103,8 @@ class DeepseekV32DecoderLayer(torch.nn.Module): quant_config=quant_config, prefix=f"{prefix}.mlp", reduce_results=False, + is_sequence_parallel=self.use_sequence_parallel, ) - self.use_sequence_parallel_moe = ( - parallel_config.use_sequence_parallel_moe - and parallel_config.pipeline_parallel_size == 1 - and isinstance(self.mlp, DeepseekV2MoE) - ) - self.tp_size = parallel_config.tensor_parallel_size self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.post_attention_layernorm = RMSNorm( config.hidden_size, eps=config.rms_norm_eps @@ -129,17 +118,11 @@ class DeepseekV32DecoderLayer(torch.nn.Module): residual: torch.Tensor | None, ) -> tuple[torch.Tensor, torch.Tensor]: full_num_tokens = positions.shape[0] - input_is_sequence_parallel = ( - self.use_sequence_parallel_moe - and residual is not None - and hidden_states.shape[0] != full_num_tokens - ) if residual is None: - # First layer: hidden_states is the (already reduced) embedding. residual = hidden_states hidden_states = self.input_layernorm(hidden_states) - elif input_is_sequence_parallel: + elif self.use_sequence_parallel: hidden_states, residual = self.input_layernorm(hidden_states, residual) else: # The previous layer's MLP/MoE output is left un-reduced; fuse its @@ -147,20 +130,13 @@ class DeepseekV32DecoderLayer(torch.nn.Module): hidden_states, residual = fused_allreduce_rms_norm( hidden_states, residual, self.input_layernorm ) - if input_is_sequence_parallel: - hidden_states = tensor_model_parallel_all_gather(hidden_states, 0) - hidden_states = hidden_states[:full_num_tokens] + if self.use_sequence_parallel: + hidden_states = sp_all_gather(hidden_states)[:full_num_tokens] # self_attn's o_proj runs reduce_results=False; reduce before RMSNorm. hidden_states = self.self_attn(positions=positions, hidden_states=hidden_states) - if self.use_sequence_parallel_moe: - # small trick using minus, eg. -17 % 8 = 7 - sp_pad = (-hidden_states.shape[0]) % self.tp_size - # pad if not divisible by world size - hidden_states = torch.nn.functional.pad(hidden_states, (0, 0, 0, sp_pad)) - hidden_states = tensor_model_parallel_reduce_scatter(hidden_states, 0) - if not input_is_sequence_parallel: - residual = sequence_parallel_chunk(residual) + if self.use_sequence_parallel: + hidden_states = sp_reduce_scatter(hidden_states) hidden_states, residual = self.post_attention_layernorm( hidden_states, residual ) @@ -169,9 +145,7 @@ class DeepseekV32DecoderLayer(torch.nn.Module): hidden_states, residual, self.post_attention_layernorm ) - # MLP/MoE runs un-reduced; its all-reduce is fused into the next layer's - # input_layernorm (or the model's final norm). - if self.use_sequence_parallel_moe: + if self.use_sequence_parallel and isinstance(self.mlp, DeepseekV2MoE): hidden_states = self.mlp(hidden_states, already_sequence_parallel=True) else: hidden_states = self.mlp(hidden_states) @@ -192,6 +166,11 @@ class DeepseekV32Model(torch.nn.Module): self.device = current_platform.device_type self.vocab_size = config.vocab_size + parallel_config = vllm_config.parallel_config + self.use_sequence_parallel = ( + parallel_config.use_sequence_parallel_moe + and parallel_config.pipeline_parallel_size == 1 + ) # DSA is always sparse (has index_topk); allocate the shared top-k # buffer the indexer writes and the sparse MLA backend reads. self.is_v32 = True @@ -231,9 +210,7 @@ class DeepseekV32Model(torch.nn.Module): ) self.aux_hidden_state_layers = tuple[int, ...]() - self.num_redundant_experts = ( - vllm_config.parallel_config.eplb_config.num_redundant_experts - ) + self.num_redundant_experts = parallel_config.eplb_config.num_redundant_experts def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.embed_tokens(input_ids) @@ -257,39 +234,49 @@ class DeepseekV32Model(torch.nn.Module): hidden_states = intermediate_tensors["hidden_states"] residual = intermediate_tensors["residual"] - aux_hidden_states = [] full_num_tokens = positions.shape[0] + if self.use_sequence_parallel: + if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available(): + forward_context = get_forward_context() + forward_context.is_padding = sp_padding_mask( + forward_context.is_padding, hidden_states + ) + hidden_states = sp_shard(hidden_states) + assert residual is None, "Currently, SP is not supported with PP" + + aux_hidden_states = [] for idx, layer in enumerate( islice(self.layers, self.start_layer, self.end_layer), start=self.start_layer, ): - if ( - hidden_states.shape[0] != full_num_tokens - and not layer.use_sequence_parallel_moe - ): - hidden_states, residual = _all_gather_sp_states( - hidden_states, residual, full_num_tokens - ) if idx in self.aux_hidden_state_layers: - aux_hidden_state = hidden_states + residual - if aux_hidden_state.shape[0] != full_num_tokens: - aux_hidden_state = tensor_model_parallel_all_gather( - aux_hidden_state, 0 - ) - aux_hidden_state = aux_hidden_state[:full_num_tokens] - aux_hidden_states.append(aux_hidden_state) + aux_hidden_states.append( + hidden_states if residual is None else hidden_states + residual + ) hidden_states, residual = layer(positions, hidden_states, residual) if not get_pp_group().is_last_rank: + assert not self.use_sequence_parallel, ( + "Currently, SP is not supported with PP" + ) return IntermediateTensors( {"hidden_states": hidden_states, "residual": residual} ) - if hidden_states.shape[0] != full_num_tokens: - hidden_states, residual = _all_gather_sp_states( - hidden_states, residual, full_num_tokens - ) + if self.use_sequence_parallel: hidden_states, _ = self.norm(hidden_states, residual) + if aux_hidden_states: + hidden_size = hidden_states.shape[-1] + packed_hidden_states = torch.cat( + [hidden_states, *aux_hidden_states], dim=-1 + ) + packed_hidden_states = sp_all_gather(packed_hidden_states) + packed_hidden_states = packed_hidden_states[:full_num_tokens] + hidden_states, *aux_hidden_states = packed_hidden_states.split( + hidden_size, dim=-1 + ) + else: + hidden_states = sp_all_gather(hidden_states)[:full_num_tokens] else: hidden_states, _ = fused_allreduce_rms_norm( hidden_states, residual, self.norm diff --git a/vllm/models/deepseek_v32/nvidia/mtp.py b/vllm/models/deepseek_v32/nvidia/mtp.py index fd2e426c63..5ef6fbf03c 100644 --- a/vllm/models/deepseek_v32/nvidia/mtp.py +++ b/vllm/models/deepseek_v32/nvidia/mtp.py @@ -6,12 +6,11 @@ from collections.abc import Callable, Iterable import torch import torch.nn as nn +import vllm.envs as envs from vllm._aiter_ops import rocm_aiter_ops from vllm.config import VllmConfig -from vllm.distributed import ( - tensor_model_parallel_all_gather, - tensor_model_parallel_all_reduce, -) +from vllm.distributed import tensor_model_parallel_all_reduce +from vllm.forward_context import get_forward_context, is_forward_context_available from vllm.model_executor.layers.fused_moe import ( fused_moe_make_expert_params_mapping, ) @@ -38,6 +37,11 @@ from vllm.model_executor.models.utils import ( get_pp_missing_layer_names, maybe_prefix, ) +from vllm.models.common.ops.sequence_parallel import ( + sp_all_gather, + sp_padding_mask, + sp_shard, +) from vllm.models.deepseek_v32.common.kernels import fused_eh_norm from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors @@ -101,13 +105,20 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module): self.hnorm.weight, self.enorm.variance_epsilon, ) + is_sequence_parallel = self.mtp_block.use_sequence_parallel + if is_sequence_parallel: + if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available(): + forward_context = get_forward_context() + forward_context.is_padding = sp_padding_mask( + forward_context.is_padding, eh_input + ) + eh_input = sp_shard(eh_input) hidden_states = run_glm52_plan(self._eh_plan, eh_input, self.eh_proj.weight) if hidden_states is None: hidden_states = self.eh_proj(eh_input) hidden_states, residual = self.mtp_block( positions=positions, hidden_states=hidden_states, residual=None ) - is_sequence_parallel = self.mtp_block.use_sequence_parallel_moe if not is_sequence_parallel: # Without sequence parallelism, the MoE output is left un-reduced. hidden_states = tensor_model_parallel_all_reduce(hidden_states) @@ -123,8 +134,7 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module): # DeepSeekMTPModel architecture). hidden_states, _ = self.shared_head.norm(hidden_states, residual) if is_sequence_parallel: - hidden_states = tensor_model_parallel_all_gather(hidden_states, 0) - hidden_states = hidden_states[: positions.shape[0]] + hidden_states = sp_all_gather(hidden_states)[: positions.shape[0]] return hidden_states, hidden_states