[DSV4] Implement Sequence Parallelism (#46789)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
from types import MethodType, SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
@@ -9,9 +10,9 @@ import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.models.common.ops import sequence_parallel as sp_ops
|
||||
from vllm.models.kimi_k3.nvidia import model as kimi_model
|
||||
from vllm.models.kimi_k3.nvidia import mtp as kimi_mtp
|
||||
from vllm.models.kimi_k3.nvidia.ops import sequence_parallel as sp_ops
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@@ -316,6 +317,23 @@ def test_sp_reduce_scatter_uses_custom_kernel_after_padding(monkeypatch):
|
||||
fallback.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", [(3,), (3, 2, 2)])
|
||||
def test_sp_shard_pads_only_the_token_axis(monkeypatch, shape):
|
||||
hidden_states = torch.arange(math.prod(shape), dtype=torch.float32).view(shape)
|
||||
monkeypatch.setattr(
|
||||
sp_ops,
|
||||
"get_tensor_model_parallel_world_size",
|
||||
lambda: 2,
|
||||
)
|
||||
monkeypatch.setattr(sp_ops, "get_tensor_model_parallel_rank", lambda: 1)
|
||||
|
||||
output = sp_ops.sp_shard(hidden_states)
|
||||
|
||||
padding = hidden_states.new_zeros((1, *shape[1:]))
|
||||
expected = torch.cat([hidden_states, padding])[2:]
|
||||
torch.testing.assert_close(output, expected)
|
||||
|
||||
|
||||
def test_sp_collectives_fall_back_without_custom_kernel(monkeypatch):
|
||||
hidden_states = torch.arange(4, dtype=torch.float32).view(2, 2)
|
||||
monkeypatch.setattr(
|
||||
|
||||
+4
-9
@@ -12,17 +12,12 @@ from vllm.distributed import (
|
||||
)
|
||||
|
||||
|
||||
def _custom_collective(
|
||||
name: str,
|
||||
x: torch.Tensor,
|
||||
) -> torch.Tensor | None:
|
||||
def _custom_collective(name: str, x: torch.Tensor) -> torch.Tensor | None:
|
||||
device_communicator = get_tp_group().device_communicator
|
||||
if device_communicator is None:
|
||||
return None
|
||||
collective = getattr(device_communicator, name, None)
|
||||
if collective is None:
|
||||
return None
|
||||
return collective(x)
|
||||
return None if collective is None else collective(x)
|
||||
|
||||
|
||||
def sp_all_gather(x: torch.Tensor) -> torch.Tensor:
|
||||
@@ -45,12 +40,12 @@ def sp_reduce_scatter(x: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
|
||||
def sp_shard(x: torch.Tensor) -> torch.Tensor:
|
||||
assert x.ndim == 2
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
sp_pad = (-x.shape[0]) % tp_size
|
||||
if sp_pad > 0:
|
||||
x = torch.nn.functional.pad(x, (0, 0, 0, sp_pad))
|
||||
pad = (0, 0) * (x.ndim - 1) + (0, sp_pad)
|
||||
x = torch.nn.functional.pad(x, pad)
|
||||
chunk = x.shape[0] // tp_size
|
||||
return x[tp_rank * chunk : (tp_rank + 1) * chunk]
|
||||
|
||||
@@ -15,11 +15,13 @@ import regex as re
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.mhc.tilelang import (
|
||||
hc_head_fused_kernel_tilelang,
|
||||
@@ -40,10 +42,16 @@ from vllm.model_executor.models.qwen3_dspark import (
|
||||
DSparkMarkovHead,
|
||||
)
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_all_gather,
|
||||
sp_padding_mask,
|
||||
sp_shard,
|
||||
)
|
||||
|
||||
from .model import (
|
||||
DeepseekV4DecoderLayer,
|
||||
DeepseekV4Model,
|
||||
_use_sequence_parallel,
|
||||
make_deepseek_v4_expert_params_mapping,
|
||||
)
|
||||
|
||||
@@ -66,6 +74,7 @@ class DSparkDeepseekV4Model(nn.Module):
|
||||
self.rms_norm_eps = config.rms_norm_eps
|
||||
self.num_hidden_layers = config.num_hidden_layers
|
||||
self.target_layer_ids = tuple(config.dspark_target_layer_ids)
|
||||
self.use_sequence_parallel = _use_sequence_parallel(vllm_config)
|
||||
|
||||
self.num_dspark_layers = getattr(config, "n_mtp_layers", None) or 3
|
||||
|
||||
@@ -86,12 +95,19 @@ class DSparkDeepseekV4Model(nn.Module):
|
||||
)
|
||||
self.main_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
self.topk_indices_buffer = torch.empty(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
config.index_topk,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
current_vllm_config = get_current_vllm_config()
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
DeepseekV4DecoderLayer(
|
||||
current_vllm_config,
|
||||
prefix=maybe_prefix(prefix, f"layers.{self.num_hidden_layers + i}"),
|
||||
topk_indices_buffer=self.topk_indices_buffer,
|
||||
)
|
||||
for i in range(self.num_dspark_layers)
|
||||
]
|
||||
@@ -172,6 +188,15 @@ class DSparkDeepseekV4Model(nn.Module):
|
||||
) -> torch.Tensor:
|
||||
if inputs_embeds is None:
|
||||
inputs_embeds = self.embed_input_ids(input_ids)
|
||||
full_num_tokens = positions.shape[0]
|
||||
if self.use_sequence_parallel:
|
||||
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
|
||||
forward_context = get_forward_context()
|
||||
forward_context.is_padding = sp_padding_mask(
|
||||
forward_context.is_padding, inputs_embeds
|
||||
)
|
||||
inputs_embeds = sp_shard(inputs_embeds)
|
||||
input_ids = sp_shard(input_ids)
|
||||
# Expand to hc_mult copies for hyper-connections ([T, H] -> [T, hc, H]).
|
||||
hidden_states = inputs_embeds.unsqueeze(-2).repeat(1, self.hc_mult, 1)
|
||||
|
||||
@@ -186,6 +211,8 @@ class DSparkDeepseekV4Model(nn.Module):
|
||||
residual,
|
||||
)
|
||||
hidden_states = mhc_post_tilelang(hidden_states, residual, post_mix, res_mix)
|
||||
if self.use_sequence_parallel:
|
||||
hidden_states = sp_all_gather(hidden_states)[:full_num_tokens]
|
||||
# hc_head reduces the hc copies; return the PRE-norm head hidden
|
||||
hidden_states = hc_head_fused_kernel_tilelang(
|
||||
hidden_states,
|
||||
@@ -279,10 +306,9 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
self.draft_model_config = vllm_config.speculative_config.draft_model_config
|
||||
self.config = self.draft_model_config.hf_config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
self.pad_shared_expert = (
|
||||
getattr(self.quant_config, "weight_block_size", None) is not None
|
||||
and not vllm_config.parallel_config.use_sequence_parallel_moe
|
||||
)
|
||||
self.pad_shared_expert = getattr(
|
||||
self.quant_config, "weight_block_size", None
|
||||
) is not None and not _use_sequence_parallel(vllm_config)
|
||||
self.model = DSparkDeepseekV4Model(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
|
||||
@@ -65,6 +65,12 @@ from vllm.model_executor.models.utils import (
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_all_gather,
|
||||
sp_padding_mask,
|
||||
sp_reduce_scatter,
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.models.deepseek_v4.attention import DeepseekV4Attention
|
||||
from vllm.models.deepseek_v4.eager_scratch import DeepseekV4EagerScratchPool
|
||||
from vllm.models.deepseek_v4.nvidia.flashinfer_sparse import (
|
||||
@@ -515,6 +521,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
use_sequence_parallel: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
@@ -522,6 +529,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
self.prefix = prefix
|
||||
self.use_sequence_parallel = use_sequence_parallel
|
||||
self.use_mega_moe = (
|
||||
vllm_config.kernel_config.moe_backend == "deep_gemm_mega_moe"
|
||||
)
|
||||
@@ -595,6 +603,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
swiglu_limit=self.swiglu_limit,
|
||||
quant_config=quant_config,
|
||||
reduce_results=self.use_mega_moe,
|
||||
is_sequence_parallel=use_sequence_parallel,
|
||||
prefix=f"{prefix}.shared_experts",
|
||||
)
|
||||
|
||||
@@ -689,6 +698,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
router_logits_dtype=torch.float32,
|
||||
enable_eplb=parallel_config.enable_eplb,
|
||||
num_redundant_experts=eplb_config.num_redundant_experts,
|
||||
is_sequence_parallel=self.use_sequence_parallel,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -792,6 +802,17 @@ def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
|
||||
return DeepseekV4FlashMLAAttention
|
||||
|
||||
|
||||
def _use_sequence_parallel(vllm_config: VllmConfig) -> bool:
|
||||
parallel_config = vllm_config.parallel_config
|
||||
use_mega_moe = vllm_config.kernel_config.moe_backend == "deep_gemm_mega_moe"
|
||||
return (
|
||||
parallel_config.pipeline_parallel_size == 1
|
||||
and parallel_config.enable_expert_parallel
|
||||
and parallel_config.tensor_parallel_size > 1
|
||||
and (use_mega_moe or parallel_config.data_parallel_size > 1)
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV4DecoderLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -805,6 +826,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.hidden_size = config.hidden_size
|
||||
self.use_sequence_parallel = _use_sequence_parallel(vllm_config)
|
||||
|
||||
self.rms_norm_eps = config.rms_norm_eps
|
||||
self.attn = _select_dsv4_attn_cls(vllm_config)(
|
||||
@@ -814,7 +836,13 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
aux_stream_list=aux_stream_list,
|
||||
eager_scratch_pool=eager_scratch_pool,
|
||||
)
|
||||
self.ffn = DeepseekV4MoE(vllm_config, prefix=f"{prefix}.ffn")
|
||||
if self.use_sequence_parallel:
|
||||
self.attn.wo_b.reduce_results = False
|
||||
self.ffn = DeepseekV4MoE(
|
||||
vllm_config,
|
||||
prefix=f"{prefix}.ffn",
|
||||
use_sequence_parallel=self.use_sequence_parallel,
|
||||
)
|
||||
|
||||
self.attn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps)
|
||||
self.ffn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps)
|
||||
@@ -932,8 +960,12 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
norm_eps=attn_norm_eps,
|
||||
)
|
||||
|
||||
# attn_norm is fused into mhc_pre_tilelang / mhc_fused_post_pre above.
|
||||
if self.use_sequence_parallel:
|
||||
x = sp_all_gather(x)[: positions.shape[0]]
|
||||
|
||||
x = self.attn(positions, x, None)
|
||||
if self.use_sequence_parallel:
|
||||
x = sp_reduce_scatter(x)
|
||||
|
||||
ffn_norm_weight = self.ffn_norm.weight.data
|
||||
ffn_norm_eps = self.ffn_norm.variance_epsilon
|
||||
@@ -972,6 +1004,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
self.use_mega_moe = (
|
||||
vllm_config.kernel_config.moe_backend == "deep_gemm_mega_moe"
|
||||
)
|
||||
self.use_sequence_parallel = _use_sequence_parallel(vllm_config)
|
||||
if self.use_mega_moe and not vllm_config.parallel_config.enable_expert_parallel:
|
||||
raise NotImplementedError(
|
||||
"DeepSeek V4 MegaMoE currently requires expert parallel. "
|
||||
@@ -1114,6 +1147,16 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
if self.use_mega_moe:
|
||||
input_ids = input_ids.to(torch.int64)
|
||||
|
||||
full_num_tokens = positions.shape[0]
|
||||
if self.use_sequence_parallel:
|
||||
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
|
||||
forward_context = get_forward_context()
|
||||
forward_context.is_padding = sp_padding_mask(
|
||||
forward_context.is_padding, hidden_states
|
||||
)
|
||||
hidden_states = sp_shard(hidden_states)
|
||||
input_ids = sp_shard(input_ids)
|
||||
|
||||
residual, post_mix, res_mix = None, None, None
|
||||
aux_hidden_states: list[torch.Tensor] = []
|
||||
final_aux_recon: torch.Tensor | None = None # avoid duplicate mhc_post call
|
||||
@@ -1134,7 +1177,10 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
aux_recon = mhc_post_tilelang(
|
||||
hidden_states, residual, post_mix, res_mix
|
||||
)
|
||||
aux_hidden_states.append(aux_recon.mean(dim=1))
|
||||
aux_hidden_state = aux_recon.mean(dim=1)
|
||||
if self.use_sequence_parallel:
|
||||
aux_hidden_state = sp_all_gather(aux_hidden_state)[:full_num_tokens]
|
||||
aux_hidden_states.append(aux_hidden_state)
|
||||
final_aux_recon = aux_recon
|
||||
if layer is not None:
|
||||
# Reuse if the last layer was captured as an aux hidden state
|
||||
@@ -1148,6 +1194,9 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors({"hidden_states": hidden_states})
|
||||
|
||||
if self.use_sequence_parallel:
|
||||
hidden_states = sp_all_gather(hidden_states)[:full_num_tokens]
|
||||
|
||||
if self._mtp_hidden_buffer is not None:
|
||||
num_tokens = hidden_states.shape[0]
|
||||
self._mtp_hidden_buffer[:num_tokens].copy_(hidden_states.flatten(1))
|
||||
@@ -1194,7 +1243,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
# ranks land on the zero pad). SP / unquantized ones need no padding.
|
||||
pad_shared_expert = (
|
||||
getattr(self.quant_config, "weight_block_size", None) is not None
|
||||
and not self.parallel_config.use_sequence_parallel_moe
|
||||
and not self.use_sequence_parallel
|
||||
)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
|
||||
@@ -18,11 +18,13 @@ import regex as re
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.mhc.tilelang import (
|
||||
hc_head_fused_kernel_tilelang,
|
||||
@@ -44,6 +46,11 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.deepseek_mtp import SharedHead
|
||||
from vllm.model_executor.models.deepseek_v2 import get_spec_layer_idx_from_weight_name
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_all_gather,
|
||||
sp_padding_mask,
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.models.deepseek_v4.common.ops import (
|
||||
fused_mtp_input_rmsnorm,
|
||||
mtp_shared_head_rmsnorm,
|
||||
@@ -53,6 +60,7 @@ from vllm.sequence import IntermediateTensors
|
||||
from .model import (
|
||||
DeepseekV4DecoderLayer,
|
||||
DeepseekV4Model,
|
||||
_use_sequence_parallel,
|
||||
make_deepseek_v4_expert_params_mapping,
|
||||
)
|
||||
|
||||
@@ -158,6 +166,14 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
|
||||
self.enorm.variance_epsilon,
|
||||
self.hc_mult,
|
||||
)
|
||||
if self.mtp_block.use_sequence_parallel:
|
||||
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
|
||||
forward_context = get_forward_context()
|
||||
forward_context.is_padding = sp_padding_mask(
|
||||
forward_context.is_padding, inputs_embeds
|
||||
)
|
||||
inputs_embeds = sp_shard(inputs_embeds)
|
||||
previous_hidden_states = sp_shard(previous_hidden_states)
|
||||
hidden_states = self.h_proj(previous_hidden_states) + self.e_proj(
|
||||
inputs_embeds
|
||||
).unsqueeze(-2)
|
||||
@@ -165,6 +181,8 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
|
||||
positions=positions, x=hidden_states, input_ids=None
|
||||
)
|
||||
hidden_states = mhc_post_tilelang(hidden_states, residual, post_mix, res_mix)
|
||||
if self.mtp_block.use_sequence_parallel:
|
||||
hidden_states = sp_all_gather(hidden_states)[: positions.shape[0]]
|
||||
# Return the flat pre-hc_head residual so it can be re-fed as the
|
||||
# next spec step's `previous_hidden_states` when
|
||||
# num_speculative_tokens > 1. hc_head is deferred to compute_logits.
|
||||
@@ -266,10 +284,9 @@ class DeepSeekV4MTP(nn.Module):
|
||||
super().__init__()
|
||||
self.config = vllm_config.model_config.hf_config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
self.pad_shared_expert = (
|
||||
getattr(self.quant_config, "weight_block_size", None) is not None
|
||||
and not vllm_config.parallel_config.use_sequence_parallel_moe
|
||||
)
|
||||
self.pad_shared_expert = getattr(
|
||||
self.quant_config, "weight_block_size", None
|
||||
) is not None and not _use_sequence_parallel(vllm_config)
|
||||
self.model = DeepSeekV4MultiTokenPredictor(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
|
||||
@@ -88,6 +88,12 @@ from vllm.model_executor.models.utils import (
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.model_executor.models.vision import is_vit_use_data_parallel
|
||||
from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_all_gather,
|
||||
sp_padding_mask,
|
||||
sp_reduce_scatter,
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.models.deepseek_v4.nvidia.model import DeepseekV4MegaMoEExperts
|
||||
from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs
|
||||
from vllm.models.kimi_k3.nvidia.kda import KimiK3DeltaAttention
|
||||
@@ -96,12 +102,6 @@ from vllm.models.kimi_k3.nvidia.low_latency_gemm import (
|
||||
)
|
||||
from vllm.models.kimi_k3.nvidia.mla import MultiHeadLatentAttention
|
||||
from vllm.models.kimi_k3.nvidia.ops import attn_res
|
||||
from vllm.models.kimi_k3.nvidia.ops.sequence_parallel import (
|
||||
sp_all_gather,
|
||||
sp_padding_mask,
|
||||
sp_reduce_scatter,
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import NestedTensors
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -27,6 +27,11 @@ from vllm.model_executor.model_loader.weight_utils import (
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.model_executor.models.utils import get_pp_missing_layer_names, maybe_prefix
|
||||
from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_all_gather,
|
||||
sp_padding_mask,
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
|
||||
|
||||
@@ -38,7 +43,6 @@ from .model import (
|
||||
get_spec_layer_idx_from_weight_name,
|
||||
make_kimi_k3_mega_moe_expert_params_mapping,
|
||||
)
|
||||
from .ops.sequence_parallel import sp_all_gather, sp_padding_mask, sp_shard
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user