diff --git a/vllm/config/parallel.py b/vllm/config/parallel.py index 7c270b0c0e..909e4418f4 100644 --- a/vllm/config/parallel.py +++ b/vllm/config/parallel.py @@ -652,7 +652,6 @@ class ParallelConfig: ) and self.enable_expert_parallel and self.tensor_parallel_size > 1 - and self.data_parallel_size > 1 ) @property diff --git a/vllm/model_executor/models/deepseek_mtp.py b/vllm/model_executor/models/deepseek_mtp.py index ff63f9c361..da7f77cae0 100644 --- a/vllm/model_executor/models/deepseek_mtp.py +++ b/vllm/model_executor/models/deepseek_mtp.py @@ -10,6 +10,7 @@ from transformers import PretrainedConfig from vllm._aiter_ops import rocm_aiter_ops from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig +from vllm.distributed import tensor_model_parallel_all_gather from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe import ( fused_moe_make_expert_params_mapping, @@ -40,6 +41,24 @@ from .utils import get_pp_missing_layer_names, maybe_prefix logger = init_logger(__name__) +def _restore_full_token_layout_if_needed( + hidden_states: torch.Tensor, + residual: torch.Tensor, + num_tokens: int, +) -> tuple[torch.Tensor, torch.Tensor]: + """Restore full token rows for the MTP proposer after SP MoE layers.""" + if hidden_states.shape[0] == num_tokens: + return hidden_states, residual + + combined_states = torch.cat([hidden_states, residual], dim=-1) + combined_states = tensor_model_parallel_all_gather(combined_states, 0) + combined_states = combined_states[:num_tokens] + hidden_states, residual = combined_states.split( + [hidden_states.shape[-1], residual.shape[-1]], dim=-1 + ) + return hidden_states, residual + + class SharedHead(nn.Module): def __init__( self, @@ -119,6 +138,11 @@ class DeepSeekMultiTokenPredictorLayer(nn.Module): hidden_states=hidden_states, residual=None, ) + hidden_states, residual = _restore_full_token_layout_if_needed( + hidden_states, + residual, + positions.shape[0], + ) hidden_states = residual + hidden_states # pre-final-norm (logits hidden) # Recycle the post-final-norm hidden into the next draft step. # compute_logits applies shared_head (== final norm) to the pre-norm diff --git a/vllm/models/deepseek_v32/nvidia/mtp.py b/vllm/models/deepseek_v32/nvidia/mtp.py index 060dfbf7bd..8edb3283d0 100644 --- a/vllm/models/deepseek_v32/nvidia/mtp.py +++ b/vllm/models/deepseek_v32/nvidia/mtp.py @@ -21,7 +21,10 @@ from vllm.model_executor.model_loader.weight_utils import ( default_weight_loader, maybe_remap_kv_scale_name, ) -from vllm.model_executor.models.deepseek_mtp import SharedHead +from vllm.model_executor.models.deepseek_mtp import ( + SharedHead, + _restore_full_token_layout_if_needed, +) from vllm.model_executor.models.deepseek_v2 import ( DeepseekV2MixtureOfExperts, DeepseekV2MoE, @@ -89,7 +92,12 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module): hidden_states, residual = self.mtp_block( positions=positions, hidden_states=hidden_states, residual=None ) - # mtp_block's MoE output is left un-reduced (reduce_results=False); the + hidden_states, residual = _restore_full_token_layout_if_needed( + hidden_states, + residual, + positions.shape[0], + ) + # mtp_block's MoE output is left un-reduced (skip_final_all_reduce); the # main model fuses that all-reduce into the next norm, but here the # recycle hidden is consumed directly, so reduce it now. hidden_states = tensor_model_parallel_all_reduce(hidden_states)