From 38a466e7b6e087d67c35e7f924c04c245423c99f Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Sat, 1 Aug 2026 10:22:33 -0700 Subject: [PATCH] [DSV4] Implement Sequence Parallelism (#46789) Signed-off-by: Woosuk Kwon --- .../models/kimi_k3/test_sequence_parallel.py | 20 ++++++- .../ops/sequence_parallel.py | 13 ++--- vllm/models/deepseek_v4/nvidia/dspark.py | 34 +++++++++-- vllm/models/deepseek_v4/nvidia/model.py | 57 +++++++++++++++++-- vllm/models/deepseek_v4/nvidia/mtp.py | 25 ++++++-- vllm/models/kimi_k3/nvidia/model.py | 12 ++-- vllm/models/kimi_k3/nvidia/mtp.py | 6 +- 7 files changed, 138 insertions(+), 29 deletions(-) rename vllm/models/{kimi_k3/nvidia => common}/ops/sequence_parallel.py (89%) diff --git a/tests/models/kimi_k3/test_sequence_parallel.py b/tests/models/kimi_k3/test_sequence_parallel.py index c65f9bd953..b61bb498cf 100644 --- a/tests/models/kimi_k3/test_sequence_parallel.py +++ b/tests/models/kimi_k3/test_sequence_parallel.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import math from types import MethodType, SimpleNamespace from unittest.mock import Mock @@ -9,9 +10,9 @@ import torch from torch import nn from vllm.config import ParallelConfig +from vllm.models.common.ops import sequence_parallel as sp_ops from vllm.models.kimi_k3.nvidia import model as kimi_model from vllm.models.kimi_k3.nvidia import mtp as kimi_mtp -from vllm.models.kimi_k3.nvidia.ops import sequence_parallel as sp_ops from vllm.platforms import current_platform @@ -316,6 +317,23 @@ def test_sp_reduce_scatter_uses_custom_kernel_after_padding(monkeypatch): fallback.assert_not_called() +@pytest.mark.parametrize("shape", [(3,), (3, 2, 2)]) +def test_sp_shard_pads_only_the_token_axis(monkeypatch, shape): + hidden_states = torch.arange(math.prod(shape), dtype=torch.float32).view(shape) + monkeypatch.setattr( + sp_ops, + "get_tensor_model_parallel_world_size", + lambda: 2, + ) + monkeypatch.setattr(sp_ops, "get_tensor_model_parallel_rank", lambda: 1) + + output = sp_ops.sp_shard(hidden_states) + + padding = hidden_states.new_zeros((1, *shape[1:])) + expected = torch.cat([hidden_states, padding])[2:] + torch.testing.assert_close(output, expected) + + def test_sp_collectives_fall_back_without_custom_kernel(monkeypatch): hidden_states = torch.arange(4, dtype=torch.float32).view(2, 2) monkeypatch.setattr( diff --git a/vllm/models/kimi_k3/nvidia/ops/sequence_parallel.py b/vllm/models/common/ops/sequence_parallel.py similarity index 89% rename from vllm/models/kimi_k3/nvidia/ops/sequence_parallel.py rename to vllm/models/common/ops/sequence_parallel.py index bb577e1ba1..8631405eae 100644 --- a/vllm/models/kimi_k3/nvidia/ops/sequence_parallel.py +++ b/vllm/models/common/ops/sequence_parallel.py @@ -12,17 +12,12 @@ from vllm.distributed import ( ) -def _custom_collective( - name: str, - x: torch.Tensor, -) -> torch.Tensor | None: +def _custom_collective(name: str, x: torch.Tensor) -> torch.Tensor | None: device_communicator = get_tp_group().device_communicator if device_communicator is None: return None collective = getattr(device_communicator, name, None) - if collective is None: - return None - return collective(x) + return None if collective is None else collective(x) def sp_all_gather(x: torch.Tensor) -> torch.Tensor: @@ -45,12 +40,12 @@ def sp_reduce_scatter(x: torch.Tensor) -> torch.Tensor: def sp_shard(x: torch.Tensor) -> torch.Tensor: - assert x.ndim == 2 tp_size = get_tensor_model_parallel_world_size() tp_rank = get_tensor_model_parallel_rank() sp_pad = (-x.shape[0]) % tp_size if sp_pad > 0: - x = torch.nn.functional.pad(x, (0, 0, 0, sp_pad)) + pad = (0, 0) * (x.ndim - 1) + (0, sp_pad) + x = torch.nn.functional.pad(x, pad) chunk = x.shape[0] // tp_size return x[tp_rank * chunk : (tp_rank + 1) * chunk] diff --git a/vllm/models/deepseek_v4/nvidia/dspark.py b/vllm/models/deepseek_v4/nvidia/dspark.py index bde088edae..5478c423a2 100644 --- a/vllm/models/deepseek_v4/nvidia/dspark.py +++ b/vllm/models/deepseek_v4/nvidia/dspark.py @@ -15,11 +15,13 @@ import regex as re import torch import torch.nn as nn +import vllm.envs as envs from vllm.config import VllmConfig, get_current_vllm_config from vllm.distributed import ( get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size, ) +from vllm.forward_context import get_forward_context, is_forward_context_available from vllm.logger import init_logger from vllm.model_executor.kernels.mhc.tilelang import ( hc_head_fused_kernel_tilelang, @@ -40,10 +42,16 @@ from vllm.model_executor.models.qwen3_dspark import ( DSparkMarkovHead, ) from vllm.model_executor.models.utils import maybe_prefix +from vllm.models.common.ops.sequence_parallel import ( + sp_all_gather, + sp_padding_mask, + sp_shard, +) from .model import ( DeepseekV4DecoderLayer, DeepseekV4Model, + _use_sequence_parallel, make_deepseek_v4_expert_params_mapping, ) @@ -66,6 +74,7 @@ class DSparkDeepseekV4Model(nn.Module): self.rms_norm_eps = config.rms_norm_eps self.num_hidden_layers = config.num_hidden_layers self.target_layer_ids = tuple(config.dspark_target_layer_ids) + self.use_sequence_parallel = _use_sequence_parallel(vllm_config) self.num_dspark_layers = getattr(config, "n_mtp_layers", None) or 3 @@ -86,12 +95,19 @@ class DSparkDeepseekV4Model(nn.Module): ) self.main_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.topk_indices_buffer = torch.empty( + vllm_config.scheduler_config.max_num_batched_tokens, + config.index_topk, + dtype=torch.int32, + ) + current_vllm_config = get_current_vllm_config() self.layers = nn.ModuleList( [ DeepseekV4DecoderLayer( current_vllm_config, prefix=maybe_prefix(prefix, f"layers.{self.num_hidden_layers + i}"), + topk_indices_buffer=self.topk_indices_buffer, ) for i in range(self.num_dspark_layers) ] @@ -172,6 +188,15 @@ class DSparkDeepseekV4Model(nn.Module): ) -> torch.Tensor: if inputs_embeds is None: inputs_embeds = self.embed_input_ids(input_ids) + full_num_tokens = positions.shape[0] + if self.use_sequence_parallel: + if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available(): + forward_context = get_forward_context() + forward_context.is_padding = sp_padding_mask( + forward_context.is_padding, inputs_embeds + ) + inputs_embeds = sp_shard(inputs_embeds) + input_ids = sp_shard(input_ids) # Expand to hc_mult copies for hyper-connections ([T, H] -> [T, hc, H]). hidden_states = inputs_embeds.unsqueeze(-2).repeat(1, self.hc_mult, 1) @@ -186,6 +211,8 @@ class DSparkDeepseekV4Model(nn.Module): residual, ) hidden_states = mhc_post_tilelang(hidden_states, residual, post_mix, res_mix) + if self.use_sequence_parallel: + hidden_states = sp_all_gather(hidden_states)[:full_num_tokens] # hc_head reduces the hc copies; return the PRE-norm head hidden hidden_states = hc_head_fused_kernel_tilelang( hidden_states, @@ -279,10 +306,9 @@ class DSparkDeepseekV4ForCausalLM(nn.Module): self.draft_model_config = vllm_config.speculative_config.draft_model_config self.config = self.draft_model_config.hf_config self.quant_config = vllm_config.quant_config - self.pad_shared_expert = ( - getattr(self.quant_config, "weight_block_size", None) is not None - and not vllm_config.parallel_config.use_sequence_parallel_moe - ) + self.pad_shared_expert = getattr( + self.quant_config, "weight_block_size", None + ) is not None and not _use_sequence_parallel(vllm_config) self.model = DSparkDeepseekV4Model( vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") ) diff --git a/vllm/models/deepseek_v4/nvidia/model.py b/vllm/models/deepseek_v4/nvidia/model.py index f442959f41..8925890a3c 100644 --- a/vllm/models/deepseek_v4/nvidia/model.py +++ b/vllm/models/deepseek_v4/nvidia/model.py @@ -65,6 +65,12 @@ from vllm.model_executor.models.utils import ( maybe_prefix, ) from vllm.model_executor.utils import set_weight_attrs +from vllm.models.common.ops.sequence_parallel import ( + sp_all_gather, + sp_padding_mask, + sp_reduce_scatter, + sp_shard, +) from vllm.models.deepseek_v4.attention import DeepseekV4Attention from vllm.models.deepseek_v4.eager_scratch import DeepseekV4EagerScratchPool from vllm.models.deepseek_v4.nvidia.flashinfer_sparse import ( @@ -515,6 +521,7 @@ class DeepseekV4MoE(nn.Module): self, vllm_config: VllmConfig, prefix: str = "", + use_sequence_parallel: bool = False, ): super().__init__() @@ -522,6 +529,7 @@ class DeepseekV4MoE(nn.Module): config = vllm_config.model_config.hf_config quant_config = vllm_config.quant_config self.prefix = prefix + self.use_sequence_parallel = use_sequence_parallel self.use_mega_moe = ( vllm_config.kernel_config.moe_backend == "deep_gemm_mega_moe" ) @@ -595,6 +603,7 @@ class DeepseekV4MoE(nn.Module): swiglu_limit=self.swiglu_limit, quant_config=quant_config, reduce_results=self.use_mega_moe, + is_sequence_parallel=use_sequence_parallel, prefix=f"{prefix}.shared_experts", ) @@ -689,6 +698,7 @@ class DeepseekV4MoE(nn.Module): router_logits_dtype=torch.float32, enable_eplb=parallel_config.enable_eplb, num_redundant_experts=eplb_config.num_redundant_experts, + is_sequence_parallel=self.use_sequence_parallel, ) def forward( @@ -792,6 +802,17 @@ def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]: return DeepseekV4FlashMLAAttention +def _use_sequence_parallel(vllm_config: VllmConfig) -> bool: + parallel_config = vllm_config.parallel_config + use_mega_moe = vllm_config.kernel_config.moe_backend == "deep_gemm_mega_moe" + return ( + parallel_config.pipeline_parallel_size == 1 + and parallel_config.enable_expert_parallel + and parallel_config.tensor_parallel_size > 1 + and (use_mega_moe or parallel_config.data_parallel_size > 1) + ) + + class DeepseekV4DecoderLayer(nn.Module): def __init__( self, @@ -805,6 +826,7 @@ class DeepseekV4DecoderLayer(nn.Module): config = vllm_config.model_config.hf_config self.hidden_size = config.hidden_size + self.use_sequence_parallel = _use_sequence_parallel(vllm_config) self.rms_norm_eps = config.rms_norm_eps self.attn = _select_dsv4_attn_cls(vllm_config)( @@ -814,7 +836,13 @@ class DeepseekV4DecoderLayer(nn.Module): aux_stream_list=aux_stream_list, eager_scratch_pool=eager_scratch_pool, ) - self.ffn = DeepseekV4MoE(vllm_config, prefix=f"{prefix}.ffn") + if self.use_sequence_parallel: + self.attn.wo_b.reduce_results = False + self.ffn = DeepseekV4MoE( + vllm_config, + prefix=f"{prefix}.ffn", + use_sequence_parallel=self.use_sequence_parallel, + ) self.attn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps) self.ffn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps) @@ -932,8 +960,12 @@ class DeepseekV4DecoderLayer(nn.Module): norm_eps=attn_norm_eps, ) - # attn_norm is fused into mhc_pre_tilelang / mhc_fused_post_pre above. + if self.use_sequence_parallel: + x = sp_all_gather(x)[: positions.shape[0]] + x = self.attn(positions, x, None) + if self.use_sequence_parallel: + x = sp_reduce_scatter(x) ffn_norm_weight = self.ffn_norm.weight.data ffn_norm_eps = self.ffn_norm.variance_epsilon @@ -972,6 +1004,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): self.use_mega_moe = ( vllm_config.kernel_config.moe_backend == "deep_gemm_mega_moe" ) + self.use_sequence_parallel = _use_sequence_parallel(vllm_config) if self.use_mega_moe and not vllm_config.parallel_config.enable_expert_parallel: raise NotImplementedError( "DeepSeek V4 MegaMoE currently requires expert parallel. " @@ -1114,6 +1147,16 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): if self.use_mega_moe: input_ids = input_ids.to(torch.int64) + full_num_tokens = positions.shape[0] + if self.use_sequence_parallel: + if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available(): + forward_context = get_forward_context() + forward_context.is_padding = sp_padding_mask( + forward_context.is_padding, hidden_states + ) + hidden_states = sp_shard(hidden_states) + input_ids = sp_shard(input_ids) + residual, post_mix, res_mix = None, None, None aux_hidden_states: list[torch.Tensor] = [] final_aux_recon: torch.Tensor | None = None # avoid duplicate mhc_post call @@ -1134,7 +1177,10 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): aux_recon = mhc_post_tilelang( hidden_states, residual, post_mix, res_mix ) - aux_hidden_states.append(aux_recon.mean(dim=1)) + aux_hidden_state = aux_recon.mean(dim=1) + if self.use_sequence_parallel: + aux_hidden_state = sp_all_gather(aux_hidden_state)[:full_num_tokens] + aux_hidden_states.append(aux_hidden_state) final_aux_recon = aux_recon if layer is not None: # Reuse if the last layer was captured as an aux hidden state @@ -1148,6 +1194,9 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): if not get_pp_group().is_last_rank: return IntermediateTensors({"hidden_states": hidden_states}) + if self.use_sequence_parallel: + hidden_states = sp_all_gather(hidden_states)[:full_num_tokens] + if self._mtp_hidden_buffer is not None: num_tokens = hidden_states.shape[0] self._mtp_hidden_buffer[:num_tokens].copy_(hidden_states.flatten(1)) @@ -1194,7 +1243,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): # ranks land on the zero pad). SP / unquantized ones need no padding. pad_shared_expert = ( getattr(self.quant_config, "weight_block_size", None) is not None - and not self.parallel_config.use_sequence_parallel_moe + and not self.use_sequence_parallel ) for name, loaded_weight in weights: diff --git a/vllm/models/deepseek_v4/nvidia/mtp.py b/vllm/models/deepseek_v4/nvidia/mtp.py index 53951196b3..eaf105205c 100644 --- a/vllm/models/deepseek_v4/nvidia/mtp.py +++ b/vllm/models/deepseek_v4/nvidia/mtp.py @@ -18,11 +18,13 @@ import regex as re import torch import torch.nn as nn +import vllm.envs as envs from vllm.config import VllmConfig from vllm.distributed import ( get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size, ) +from vllm.forward_context import get_forward_context, is_forward_context_available from vllm.logger import init_logger from vllm.model_executor.kernels.mhc.tilelang import ( hc_head_fused_kernel_tilelang, @@ -44,6 +46,11 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.deepseek_mtp import SharedHead from vllm.model_executor.models.deepseek_v2 import get_spec_layer_idx_from_weight_name from vllm.model_executor.models.utils import maybe_prefix +from vllm.models.common.ops.sequence_parallel import ( + sp_all_gather, + sp_padding_mask, + sp_shard, +) from vllm.models.deepseek_v4.common.ops import ( fused_mtp_input_rmsnorm, mtp_shared_head_rmsnorm, @@ -53,6 +60,7 @@ from vllm.sequence import IntermediateTensors from .model import ( DeepseekV4DecoderLayer, DeepseekV4Model, + _use_sequence_parallel, make_deepseek_v4_expert_params_mapping, ) @@ -158,6 +166,14 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module): self.enorm.variance_epsilon, self.hc_mult, ) + if self.mtp_block.use_sequence_parallel: + if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available(): + forward_context = get_forward_context() + forward_context.is_padding = sp_padding_mask( + forward_context.is_padding, inputs_embeds + ) + inputs_embeds = sp_shard(inputs_embeds) + previous_hidden_states = sp_shard(previous_hidden_states) hidden_states = self.h_proj(previous_hidden_states) + self.e_proj( inputs_embeds ).unsqueeze(-2) @@ -165,6 +181,8 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module): positions=positions, x=hidden_states, input_ids=None ) hidden_states = mhc_post_tilelang(hidden_states, residual, post_mix, res_mix) + if self.mtp_block.use_sequence_parallel: + hidden_states = sp_all_gather(hidden_states)[: positions.shape[0]] # Return the flat pre-hc_head residual so it can be re-fed as the # next spec step's `previous_hidden_states` when # num_speculative_tokens > 1. hc_head is deferred to compute_logits. @@ -266,10 +284,9 @@ class DeepSeekV4MTP(nn.Module): super().__init__() self.config = vllm_config.model_config.hf_config self.quant_config = vllm_config.quant_config - self.pad_shared_expert = ( - getattr(self.quant_config, "weight_block_size", None) is not None - and not vllm_config.parallel_config.use_sequence_parallel_moe - ) + self.pad_shared_expert = getattr( + self.quant_config, "weight_block_size", None + ) is not None and not _use_sequence_parallel(vllm_config) self.model = DeepSeekV4MultiTokenPredictor( vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") ) diff --git a/vllm/models/kimi_k3/nvidia/model.py b/vllm/models/kimi_k3/nvidia/model.py index b65f4fdfad..94178c9c73 100644 --- a/vllm/models/kimi_k3/nvidia/model.py +++ b/vllm/models/kimi_k3/nvidia/model.py @@ -88,6 +88,12 @@ from vllm.model_executor.models.utils import ( maybe_prefix, ) from vllm.model_executor.models.vision import is_vit_use_data_parallel +from vllm.models.common.ops.sequence_parallel import ( + sp_all_gather, + sp_padding_mask, + sp_reduce_scatter, + sp_shard, +) from vllm.models.deepseek_v4.nvidia.model import DeepseekV4MegaMoEExperts from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs from vllm.models.kimi_k3.nvidia.kda import KimiK3DeltaAttention @@ -96,12 +102,6 @@ from vllm.models.kimi_k3.nvidia.low_latency_gemm import ( ) from vllm.models.kimi_k3.nvidia.mla import MultiHeadLatentAttention from vllm.models.kimi_k3.nvidia.ops import attn_res -from vllm.models.kimi_k3.nvidia.ops.sequence_parallel import ( - sp_all_gather, - sp_padding_mask, - sp_reduce_scatter, - sp_shard, -) from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import NestedTensors from vllm.platforms import current_platform diff --git a/vllm/models/kimi_k3/nvidia/mtp.py b/vllm/models/kimi_k3/nvidia/mtp.py index f08d9304b4..45cbe3767e 100644 --- a/vllm/models/kimi_k3/nvidia/mtp.py +++ b/vllm/models/kimi_k3/nvidia/mtp.py @@ -27,6 +27,11 @@ from vllm.model_executor.model_loader.weight_utils import ( maybe_remap_kv_scale_name, ) from vllm.model_executor.models.utils import get_pp_missing_layer_names, maybe_prefix +from vllm.models.common.ops.sequence_parallel import ( + sp_all_gather, + sp_padding_mask, + sp_shard, +) from vllm.sequence import IntermediateTensors from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig @@ -38,7 +43,6 @@ from .model import ( get_spec_layer_idx_from_weight_name, make_kimi_k3_mega_moe_expert_params_mapping, ) -from .ops.sequence_parallel import sp_all_gather, sp_padding_mask, sp_shard logger = init_logger(__name__)