[CI Bug] Fully solve accuracy issue for DSv3.2 + MTP + Sequence Parallel (#48036)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -652,7 +652,6 @@ class ParallelConfig:
|
||||
)
|
||||
and self.enable_expert_parallel
|
||||
and self.tensor_parallel_size > 1
|
||||
and self.data_parallel_size > 1
|
||||
)
|
||||
|
||||
@property
|
||||
|
||||
@@ -10,6 +10,7 @@ from transformers import PretrainedConfig
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import tensor_model_parallel_all_gather
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
@@ -40,6 +41,24 @@ from .utils import get_pp_missing_layer_names, maybe_prefix
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _restore_full_token_layout_if_needed(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
num_tokens: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Restore full token rows for the MTP proposer after SP MoE layers."""
|
||||
if hidden_states.shape[0] == num_tokens:
|
||||
return hidden_states, residual
|
||||
|
||||
combined_states = torch.cat([hidden_states, residual], dim=-1)
|
||||
combined_states = tensor_model_parallel_all_gather(combined_states, 0)
|
||||
combined_states = combined_states[:num_tokens]
|
||||
hidden_states, residual = combined_states.split(
|
||||
[hidden_states.shape[-1], residual.shape[-1]], dim=-1
|
||||
)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
class SharedHead(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -119,6 +138,11 @@ class DeepSeekMultiTokenPredictorLayer(nn.Module):
|
||||
hidden_states=hidden_states,
|
||||
residual=None,
|
||||
)
|
||||
hidden_states, residual = _restore_full_token_layout_if_needed(
|
||||
hidden_states,
|
||||
residual,
|
||||
positions.shape[0],
|
||||
)
|
||||
hidden_states = residual + hidden_states # pre-final-norm (logits hidden)
|
||||
# Recycle the post-final-norm hidden into the next draft step.
|
||||
# compute_logits applies shared_head (== final norm) to the pre-norm
|
||||
|
||||
@@ -21,7 +21,10 @@ from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.model_executor.models.deepseek_mtp import SharedHead
|
||||
from vllm.model_executor.models.deepseek_mtp import (
|
||||
SharedHead,
|
||||
_restore_full_token_layout_if_needed,
|
||||
)
|
||||
from vllm.model_executor.models.deepseek_v2 import (
|
||||
DeepseekV2MixtureOfExperts,
|
||||
DeepseekV2MoE,
|
||||
@@ -89,7 +92,12 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
|
||||
hidden_states, residual = self.mtp_block(
|
||||
positions=positions, hidden_states=hidden_states, residual=None
|
||||
)
|
||||
# mtp_block's MoE output is left un-reduced (reduce_results=False); the
|
||||
hidden_states, residual = _restore_full_token_layout_if_needed(
|
||||
hidden_states,
|
||||
residual,
|
||||
positions.shape[0],
|
||||
)
|
||||
# mtp_block's MoE output is left un-reduced (skip_final_all_reduce); the
|
||||
# main model fuses that all-reduce into the next norm, but here the
|
||||
# recycle hidden is consumed directly, so reduce it now.
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
|
||||
Reference in New Issue
Block a user