[DSV4] Implement Sequence Parallelism (#46789)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
Woosuk Kwon
2026-08-01 10:22:33 -07:00
committed by GitHub
parent dc818c198d
commit 38a466e7b6
7 changed files with 138 additions and 29 deletions
+19 -1
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from types import MethodType, SimpleNamespace
from unittest.mock import Mock
@@ -9,9 +10,9 @@ import torch
from torch import nn
from vllm.config import ParallelConfig
from vllm.models.common.ops import sequence_parallel as sp_ops
from vllm.models.kimi_k3.nvidia import model as kimi_model
from vllm.models.kimi_k3.nvidia import mtp as kimi_mtp
from vllm.models.kimi_k3.nvidia.ops import sequence_parallel as sp_ops
from vllm.platforms import current_platform
@@ -316,6 +317,23 @@ def test_sp_reduce_scatter_uses_custom_kernel_after_padding(monkeypatch):
fallback.assert_not_called()
@pytest.mark.parametrize("shape", [(3,), (3, 2, 2)])
def test_sp_shard_pads_only_the_token_axis(monkeypatch, shape):
hidden_states = torch.arange(math.prod(shape), dtype=torch.float32).view(shape)
monkeypatch.setattr(
sp_ops,
"get_tensor_model_parallel_world_size",
lambda: 2,
)
monkeypatch.setattr(sp_ops, "get_tensor_model_parallel_rank", lambda: 1)
output = sp_ops.sp_shard(hidden_states)
padding = hidden_states.new_zeros((1, *shape[1:]))
expected = torch.cat([hidden_states, padding])[2:]
torch.testing.assert_close(output, expected)
def test_sp_collectives_fall_back_without_custom_kernel(monkeypatch):
hidden_states = torch.arange(4, dtype=torch.float32).view(2, 2)
monkeypatch.setattr(
@@ -12,17 +12,12 @@ from vllm.distributed import (
)
def _custom_collective(
name: str,
x: torch.Tensor,
) -> torch.Tensor | None:
def _custom_collective(name: str, x: torch.Tensor) -> torch.Tensor | None:
device_communicator = get_tp_group().device_communicator
if device_communicator is None:
return None
collective = getattr(device_communicator, name, None)
if collective is None:
return None
return collective(x)
return None if collective is None else collective(x)
def sp_all_gather(x: torch.Tensor) -> torch.Tensor:
@@ -45,12 +40,12 @@ def sp_reduce_scatter(x: torch.Tensor) -> torch.Tensor:
def sp_shard(x: torch.Tensor) -> torch.Tensor:
assert x.ndim == 2
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
sp_pad = (-x.shape[0]) % tp_size
if sp_pad > 0:
x = torch.nn.functional.pad(x, (0, 0, 0, sp_pad))
pad = (0, 0) * (x.ndim - 1) + (0, sp_pad)
x = torch.nn.functional.pad(x, pad)
chunk = x.shape[0] // tp_size
return x[tp_rank * chunk : (tp_rank + 1) * chunk]
+30 -4
View File
@@ -15,11 +15,13 @@ import regex as re
import torch
import torch.nn as nn
import vllm.envs as envs
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.logger import init_logger
from vllm.model_executor.kernels.mhc.tilelang import (
hc_head_fused_kernel_tilelang,
@@ -40,10 +42,16 @@ from vllm.model_executor.models.qwen3_dspark import (
DSparkMarkovHead,
)
from vllm.model_executor.models.utils import maybe_prefix
from vllm.models.common.ops.sequence_parallel import (
sp_all_gather,
sp_padding_mask,
sp_shard,
)
from .model import (
DeepseekV4DecoderLayer,
DeepseekV4Model,
_use_sequence_parallel,
make_deepseek_v4_expert_params_mapping,
)
@@ -66,6 +74,7 @@ class DSparkDeepseekV4Model(nn.Module):
self.rms_norm_eps = config.rms_norm_eps
self.num_hidden_layers = config.num_hidden_layers
self.target_layer_ids = tuple(config.dspark_target_layer_ids)
self.use_sequence_parallel = _use_sequence_parallel(vllm_config)
self.num_dspark_layers = getattr(config, "n_mtp_layers", None) or 3
@@ -86,12 +95,19 @@ class DSparkDeepseekV4Model(nn.Module):
)
self.main_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.topk_indices_buffer = torch.empty(
vllm_config.scheduler_config.max_num_batched_tokens,
config.index_topk,
dtype=torch.int32,
)
current_vllm_config = get_current_vllm_config()
self.layers = nn.ModuleList(
[
DeepseekV4DecoderLayer(
current_vllm_config,
prefix=maybe_prefix(prefix, f"layers.{self.num_hidden_layers + i}"),
topk_indices_buffer=self.topk_indices_buffer,
)
for i in range(self.num_dspark_layers)
]
@@ -172,6 +188,15 @@ class DSparkDeepseekV4Model(nn.Module):
) -> torch.Tensor:
if inputs_embeds is None:
inputs_embeds = self.embed_input_ids(input_ids)
full_num_tokens = positions.shape[0]
if self.use_sequence_parallel:
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
forward_context = get_forward_context()
forward_context.is_padding = sp_padding_mask(
forward_context.is_padding, inputs_embeds
)
inputs_embeds = sp_shard(inputs_embeds)
input_ids = sp_shard(input_ids)
# Expand to hc_mult copies for hyper-connections ([T, H] -> [T, hc, H]).
hidden_states = inputs_embeds.unsqueeze(-2).repeat(1, self.hc_mult, 1)
@@ -186,6 +211,8 @@ class DSparkDeepseekV4Model(nn.Module):
residual,
)
hidden_states = mhc_post_tilelang(hidden_states, residual, post_mix, res_mix)
if self.use_sequence_parallel:
hidden_states = sp_all_gather(hidden_states)[:full_num_tokens]
# hc_head reduces the hc copies; return the PRE-norm head hidden
hidden_states = hc_head_fused_kernel_tilelang(
hidden_states,
@@ -279,10 +306,9 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
self.draft_model_config = vllm_config.speculative_config.draft_model_config
self.config = self.draft_model_config.hf_config
self.quant_config = vllm_config.quant_config
self.pad_shared_expert = (
getattr(self.quant_config, "weight_block_size", None) is not None
and not vllm_config.parallel_config.use_sequence_parallel_moe
)
self.pad_shared_expert = getattr(
self.quant_config, "weight_block_size", None
) is not None and not _use_sequence_parallel(vllm_config)
self.model = DSparkDeepseekV4Model(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
+53 -4
View File
@@ -65,6 +65,12 @@ from vllm.model_executor.models.utils import (
maybe_prefix,
)
from vllm.model_executor.utils import set_weight_attrs
from vllm.models.common.ops.sequence_parallel import (
sp_all_gather,
sp_padding_mask,
sp_reduce_scatter,
sp_shard,
)
from vllm.models.deepseek_v4.attention import DeepseekV4Attention
from vllm.models.deepseek_v4.eager_scratch import DeepseekV4EagerScratchPool
from vllm.models.deepseek_v4.nvidia.flashinfer_sparse import (
@@ -515,6 +521,7 @@ class DeepseekV4MoE(nn.Module):
self,
vllm_config: VllmConfig,
prefix: str = "",
use_sequence_parallel: bool = False,
):
super().__init__()
@@ -522,6 +529,7 @@ class DeepseekV4MoE(nn.Module):
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
self.prefix = prefix
self.use_sequence_parallel = use_sequence_parallel
self.use_mega_moe = (
vllm_config.kernel_config.moe_backend == "deep_gemm_mega_moe"
)
@@ -595,6 +603,7 @@ class DeepseekV4MoE(nn.Module):
swiglu_limit=self.swiglu_limit,
quant_config=quant_config,
reduce_results=self.use_mega_moe,
is_sequence_parallel=use_sequence_parallel,
prefix=f"{prefix}.shared_experts",
)
@@ -689,6 +698,7 @@ class DeepseekV4MoE(nn.Module):
router_logits_dtype=torch.float32,
enable_eplb=parallel_config.enable_eplb,
num_redundant_experts=eplb_config.num_redundant_experts,
is_sequence_parallel=self.use_sequence_parallel,
)
def forward(
@@ -792,6 +802,17 @@ def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
return DeepseekV4FlashMLAAttention
def _use_sequence_parallel(vllm_config: VllmConfig) -> bool:
parallel_config = vllm_config.parallel_config
use_mega_moe = vllm_config.kernel_config.moe_backend == "deep_gemm_mega_moe"
return (
parallel_config.pipeline_parallel_size == 1
and parallel_config.enable_expert_parallel
and parallel_config.tensor_parallel_size > 1
and (use_mega_moe or parallel_config.data_parallel_size > 1)
)
class DeepseekV4DecoderLayer(nn.Module):
def __init__(
self,
@@ -805,6 +826,7 @@ class DeepseekV4DecoderLayer(nn.Module):
config = vllm_config.model_config.hf_config
self.hidden_size = config.hidden_size
self.use_sequence_parallel = _use_sequence_parallel(vllm_config)
self.rms_norm_eps = config.rms_norm_eps
self.attn = _select_dsv4_attn_cls(vllm_config)(
@@ -814,7 +836,13 @@ class DeepseekV4DecoderLayer(nn.Module):
aux_stream_list=aux_stream_list,
eager_scratch_pool=eager_scratch_pool,
)
self.ffn = DeepseekV4MoE(vllm_config, prefix=f"{prefix}.ffn")
if self.use_sequence_parallel:
self.attn.wo_b.reduce_results = False
self.ffn = DeepseekV4MoE(
vllm_config,
prefix=f"{prefix}.ffn",
use_sequence_parallel=self.use_sequence_parallel,
)
self.attn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps)
self.ffn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps)
@@ -932,8 +960,12 @@ class DeepseekV4DecoderLayer(nn.Module):
norm_eps=attn_norm_eps,
)
# attn_norm is fused into mhc_pre_tilelang / mhc_fused_post_pre above.
if self.use_sequence_parallel:
x = sp_all_gather(x)[: positions.shape[0]]
x = self.attn(positions, x, None)
if self.use_sequence_parallel:
x = sp_reduce_scatter(x)
ffn_norm_weight = self.ffn_norm.weight.data
ffn_norm_eps = self.ffn_norm.variance_epsilon
@@ -972,6 +1004,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
self.use_mega_moe = (
vllm_config.kernel_config.moe_backend == "deep_gemm_mega_moe"
)
self.use_sequence_parallel = _use_sequence_parallel(vllm_config)
if self.use_mega_moe and not vllm_config.parallel_config.enable_expert_parallel:
raise NotImplementedError(
"DeepSeek V4 MegaMoE currently requires expert parallel. "
@@ -1114,6 +1147,16 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
if self.use_mega_moe:
input_ids = input_ids.to(torch.int64)
full_num_tokens = positions.shape[0]
if self.use_sequence_parallel:
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
forward_context = get_forward_context()
forward_context.is_padding = sp_padding_mask(
forward_context.is_padding, hidden_states
)
hidden_states = sp_shard(hidden_states)
input_ids = sp_shard(input_ids)
residual, post_mix, res_mix = None, None, None
aux_hidden_states: list[torch.Tensor] = []
final_aux_recon: torch.Tensor | None = None # avoid duplicate mhc_post call
@@ -1134,7 +1177,10 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
aux_recon = mhc_post_tilelang(
hidden_states, residual, post_mix, res_mix
)
aux_hidden_states.append(aux_recon.mean(dim=1))
aux_hidden_state = aux_recon.mean(dim=1)
if self.use_sequence_parallel:
aux_hidden_state = sp_all_gather(aux_hidden_state)[:full_num_tokens]
aux_hidden_states.append(aux_hidden_state)
final_aux_recon = aux_recon
if layer is not None:
# Reuse if the last layer was captured as an aux hidden state
@@ -1148,6 +1194,9 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
if not get_pp_group().is_last_rank:
return IntermediateTensors({"hidden_states": hidden_states})
if self.use_sequence_parallel:
hidden_states = sp_all_gather(hidden_states)[:full_num_tokens]
if self._mtp_hidden_buffer is not None:
num_tokens = hidden_states.shape[0]
self._mtp_hidden_buffer[:num_tokens].copy_(hidden_states.flatten(1))
@@ -1194,7 +1243,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
# ranks land on the zero pad). SP / unquantized ones need no padding.
pad_shared_expert = (
getattr(self.quant_config, "weight_block_size", None) is not None
and not self.parallel_config.use_sequence_parallel_moe
and not self.use_sequence_parallel
)
for name, loaded_weight in weights:
+21 -4
View File
@@ -18,11 +18,13 @@ import regex as re
import torch
import torch.nn as nn
import vllm.envs as envs
from vllm.config import VllmConfig
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.logger import init_logger
from vllm.model_executor.kernels.mhc.tilelang import (
hc_head_fused_kernel_tilelang,
@@ -44,6 +46,11 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.deepseek_mtp import SharedHead
from vllm.model_executor.models.deepseek_v2 import get_spec_layer_idx_from_weight_name
from vllm.model_executor.models.utils import maybe_prefix
from vllm.models.common.ops.sequence_parallel import (
sp_all_gather,
sp_padding_mask,
sp_shard,
)
from vllm.models.deepseek_v4.common.ops import (
fused_mtp_input_rmsnorm,
mtp_shared_head_rmsnorm,
@@ -53,6 +60,7 @@ from vllm.sequence import IntermediateTensors
from .model import (
DeepseekV4DecoderLayer,
DeepseekV4Model,
_use_sequence_parallel,
make_deepseek_v4_expert_params_mapping,
)
@@ -158,6 +166,14 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
self.enorm.variance_epsilon,
self.hc_mult,
)
if self.mtp_block.use_sequence_parallel:
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
forward_context = get_forward_context()
forward_context.is_padding = sp_padding_mask(
forward_context.is_padding, inputs_embeds
)
inputs_embeds = sp_shard(inputs_embeds)
previous_hidden_states = sp_shard(previous_hidden_states)
hidden_states = self.h_proj(previous_hidden_states) + self.e_proj(
inputs_embeds
).unsqueeze(-2)
@@ -165,6 +181,8 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
positions=positions, x=hidden_states, input_ids=None
)
hidden_states = mhc_post_tilelang(hidden_states, residual, post_mix, res_mix)
if self.mtp_block.use_sequence_parallel:
hidden_states = sp_all_gather(hidden_states)[: positions.shape[0]]
# Return the flat pre-hc_head residual so it can be re-fed as the
# next spec step's `previous_hidden_states` when
# num_speculative_tokens > 1. hc_head is deferred to compute_logits.
@@ -266,10 +284,9 @@ class DeepSeekV4MTP(nn.Module):
super().__init__()
self.config = vllm_config.model_config.hf_config
self.quant_config = vllm_config.quant_config
self.pad_shared_expert = (
getattr(self.quant_config, "weight_block_size", None) is not None
and not vllm_config.parallel_config.use_sequence_parallel_moe
)
self.pad_shared_expert = getattr(
self.quant_config, "weight_block_size", None
) is not None and not _use_sequence_parallel(vllm_config)
self.model = DeepSeekV4MultiTokenPredictor(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
+6 -6
View File
@@ -88,6 +88,12 @@ from vllm.model_executor.models.utils import (
maybe_prefix,
)
from vllm.model_executor.models.vision import is_vit_use_data_parallel
from vllm.models.common.ops.sequence_parallel import (
sp_all_gather,
sp_padding_mask,
sp_reduce_scatter,
sp_shard,
)
from vllm.models.deepseek_v4.nvidia.model import DeepseekV4MegaMoEExperts
from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs
from vllm.models.kimi_k3.nvidia.kda import KimiK3DeltaAttention
@@ -96,12 +102,6 @@ from vllm.models.kimi_k3.nvidia.low_latency_gemm import (
)
from vllm.models.kimi_k3.nvidia.mla import MultiHeadLatentAttention
from vllm.models.kimi_k3.nvidia.ops import attn_res
from vllm.models.kimi_k3.nvidia.ops.sequence_parallel import (
sp_all_gather,
sp_padding_mask,
sp_reduce_scatter,
sp_shard,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import NestedTensors
from vllm.platforms import current_platform
+5 -1
View File
@@ -27,6 +27,11 @@ from vllm.model_executor.model_loader.weight_utils import (
maybe_remap_kv_scale_name,
)
from vllm.model_executor.models.utils import get_pp_missing_layer_names, maybe_prefix
from vllm.models.common.ops.sequence_parallel import (
sp_all_gather,
sp_padding_mask,
sp_shard,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
@@ -38,7 +43,6 @@ from .model import (
get_spec_layer_idx_from_weight_name,
make_kimi_k3_mega_moe_expert_params_mapping,
)
from .ops.sequence_parallel import sp_all_gather, sp_padding_mask, sp_shard
logger = init_logger(__name__)