From c05d75aaa95cf89f547503044c1921625905085d Mon Sep 17 00:00:00 2001 From: kzwrime <66299956+kzwrime@users.noreply.github.com> Date: Fri, 14 Aug 2026 11:58:32 +0800 Subject: [PATCH] [Bugfix][Refactor] Keep Qwen3Next layer boundaries sequence parallel (#50685) Signed-off-by: kunzh Signed-off-by: zjy0516 Co-authored-by: OpenAI Codex Co-authored-by: zjy0516 --- vllm/model_executor/models/interns2_mobius.py | 57 +++++------- vllm/model_executor/models/qwen3_5_mtp.py | 19 ++-- vllm/model_executor/models/qwen3_next.py | 92 +++++++------------ vllm/model_executor/models/qwen3_next_mtp.py | 18 ++-- 4 files changed, 75 insertions(+), 111 deletions(-) diff --git a/vllm/model_executor/models/interns2_mobius.py b/vllm/model_executor/models/interns2_mobius.py index 1bbc614334..b8111412f0 100644 --- a/vllm/model_executor/models/interns2_mobius.py +++ b/vllm/model_executor/models/interns2_mobius.py @@ -43,7 +43,7 @@ from .qwen3_5 import ( Qwen3_5RMSNorm, ) from .qwen3_5_mtp import Qwen3_5MoeMTP -from .qwen3_next import Qwen3NextAttention, _all_gather_hidden_and_residual +from .qwen3_next import Qwen3NextAttention from .qwen3_vl import ( Qwen3_VisionTransformer, Qwen3VLDummyInputsBuilder, @@ -233,11 +233,6 @@ class InternS2MobiusDecoderLayer(nn.Module): meta_mlp: nn.ModuleList, ) -> tuple[torch.Tensor, torch.Tensor]: full_num_tokens = positions.shape[-1] - input_is_sequence_parallel = ( - self.use_attn_reduce_scatter_for_moe - and residual is not None - and hidden_states.shape[0] != full_num_tokens - ) if residual is None: residual = hidden_states @@ -245,7 +240,7 @@ class InternS2MobiusDecoderLayer(nn.Module): else: hidden_states, residual = self.input_layernorm(hidden_states, residual) - if input_is_sequence_parallel: + if self.use_attn_reduce_scatter_for_moe: hidden_states = tensor_model_parallel_all_gather(hidden_states, 0) hidden_states = hidden_states[:full_num_tokens] @@ -266,8 +261,6 @@ class InternS2MobiusDecoderLayer(nn.Module): sp_pad = (-hidden_states.shape[0]) % tp_world_size hidden_states = torch.nn.functional.pad(hidden_states, (0, 0, 0, sp_pad)) hidden_states = tensor_model_parallel_reduce_scatter(hidden_states, 0) - if not input_is_sequence_parallel: - residual = sequence_parallel_chunk(residual) hidden_states, residual = self.post_attention_layernorm(hidden_states, residual) block_idx = self.layer_idx % self.num_blocks @@ -343,6 +336,10 @@ class InternS2MobiusModel(nn.Module, EagleModelMixin): def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.embed_tokens(input_ids) + @property + def use_sequence_parallel(self) -> bool: + return self.layers[self.start_layer].use_attn_reduce_scatter_for_moe + def forward( self, input_ids: torch.Tensor | None, @@ -363,36 +360,21 @@ class InternS2MobiusModel(nn.Module, EagleModelMixin): residual = intermediate_tensors["residual"] full_num_tokens = positions.shape[-1] + if self.use_sequence_parallel: + hidden_states = sequence_parallel_chunk(hidden_states) + assert residual is None + aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual) for layer_idx, layer in enumerate( islice(self.layers, self.start_layer, self.end_layer), start=self.start_layer, ): - if ( - hidden_states.shape[0] != full_num_tokens - and not layer.use_attn_reduce_scatter_for_moe - ): - hidden_states, residual = _all_gather_hidden_and_residual( - hidden_states, - residual, - full_num_tokens, - self.config.hidden_size, - ) hidden_states, residual = layer( positions=positions, hidden_states=hidden_states, residual=residual, meta_mlp=self.meta_mlp, ) - if (layer_idx + 1) in self.aux_hidden_state_layers and hidden_states.shape[ - 0 - ] != full_num_tokens: - hidden_states, residual = _all_gather_hidden_and_residual( - hidden_states, - residual, - full_num_tokens, - self.config.hidden_size, - ) self._maybe_add_hidden_state( aux_hidden_states, layer_idx + 1, @@ -404,14 +386,19 @@ class InternS2MobiusModel(nn.Module, EagleModelMixin): return IntermediateTensors( {"hidden_states": hidden_states, "residual": residual} ) - if hidden_states.shape[0] != full_num_tokens: - hidden_states, residual = _all_gather_hidden_and_residual( - hidden_states, - residual, - full_num_tokens, - self.config.hidden_size, - ) hidden_states, _ = self.norm(hidden_states, residual) + if self.use_sequence_parallel: + if aux_hidden_states: + hidden_size = hidden_states.shape[-1] + hidden_states = torch.cat([hidden_states, *aux_hidden_states], dim=-1) + hidden_states = tensor_model_parallel_all_gather(hidden_states, 0) + hidden_states = hidden_states[:full_num_tokens] + hidden_states, *aux_hidden_states = hidden_states.split( + hidden_size, dim=-1 + ) + else: + hidden_states = tensor_model_parallel_all_gather(hidden_states, 0) + hidden_states = hidden_states[:full_num_tokens] if aux_hidden_states: return hidden_states, aux_hidden_states return hidden_states diff --git a/vllm/model_executor/models/qwen3_5_mtp.py b/vllm/model_executor/models/qwen3_5_mtp.py index 7dc7c53826..67f06f4f5f 100644 --- a/vllm/model_executor/models/qwen3_5_mtp.py +++ b/vllm/model_executor/models/qwen3_5_mtp.py @@ -10,7 +10,7 @@ from torch import nn from vllm._aiter_ops import rocm_aiter_ops from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig, get_current_vllm_config -from vllm.distributed.parallel_state import get_pp_group +from vllm.distributed import get_pp_group, tensor_model_parallel_all_gather from vllm.logger import init_logger from vllm.model_executor.layers.linear import ColumnParallelLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor @@ -26,9 +26,9 @@ from vllm.model_executor.models.qwen3_5 import ( ) from vllm.model_executor.models.qwen3_next import ( QwenNextMixtureOfExperts, - _all_gather_hidden_and_residual, _is_shared_expert_fse_compatible, ) +from vllm.model_executor.models.utils import sequence_parallel_chunk from vllm.sequence import IntermediateTensors from vllm.transformers_utils.configs.qwen3_5 import Qwen3_5TextConfig from vllm.transformers_utils.configs.qwen3_5_moe import Qwen3_5MoeTextConfig @@ -161,6 +161,10 @@ class Qwen3_5MultiTokenPredictor(nn.Module): current_step_idx = spec_step_idx % self.num_mtp_layers mtp_layer = self.layers[current_step_idx] + if mtp_layer.use_attn_reduce_scatter_for_moe: + assert hidden_states.shape[0] == positions.shape[-1] + hidden_states = sequence_parallel_chunk(hidden_states) + assert residual is None hidden_states, residual = mtp_layer( positions=positions, hidden_states=hidden_states, @@ -172,15 +176,10 @@ class Qwen3_5MultiTokenPredictor(nn.Module): {"hidden_states": hidden_states, "residual": residual} ) - if mtp_layer.use_attn_reduce_scatter_for_moe: - hidden_states, residual = _all_gather_hidden_and_residual( - hidden_states, - residual, - positions.shape[-1], - self.config.hidden_size, - ) - hidden_states, _ = self.norm(hidden_states, residual) + if mtp_layer.use_attn_reduce_scatter_for_moe: + hidden_states = tensor_model_parallel_all_gather(hidden_states, 0) + hidden_states = hidden_states[: positions.shape[-1]] return hidden_states def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: diff --git a/vllm/model_executor/models/qwen3_next.py b/vllm/model_executor/models/qwen3_next.py index ffeada5444..b241baca6c 100644 --- a/vllm/model_executor/models/qwen3_next.py +++ b/vllm/model_executor/models/qwen3_next.py @@ -78,6 +78,18 @@ logger = init_logger(__name__) KVCache = tuple[torch.Tensor, torch.Tensor] +def _should_use_sequence_parallel(vllm_config: VllmConfig) -> bool: + config = vllm_config.model_config.hf_text_config + parallel_config = vllm_config.parallel_config + return ( + parallel_config.use_sequence_parallel_moe + and parallel_config.pipeline_parallel_size == 1 + and getattr(config, "num_experts", 0) > 0 + and not getattr(config, "mlp_only_layers", []) + and getattr(config, "decoder_sparse_step", 1) == 1 + ) + + def _is_shared_expert_fse_compatible(quant_config) -> bool: """Check if shared expert can be fused with routed experts. @@ -397,7 +409,6 @@ class Qwen3NextDecoderLayer(nn.Module): model_config = vllm_config.model_config cache_config = vllm_config.cache_config quant_config = vllm_config.quant_config - parallel_config = vllm_config.parallel_config self.layer_type = layer_type self.layer_idx = extract_layer_index(prefix) @@ -409,10 +420,8 @@ class Qwen3NextDecoderLayer(nn.Module): config.num_experts > 0 and (self.layer_idx + 1) % config.decoder_sparse_step == 0 ) - self.use_attn_reduce_scatter_for_moe = ( - parallel_config.use_sequence_parallel_moe - and parallel_config.pipeline_parallel_size == 1 - and is_moe_layer + self.use_attn_reduce_scatter_for_moe = _should_use_sequence_parallel( + vllm_config ) if self.layer_type == "linear_attention": @@ -481,11 +490,6 @@ class Qwen3NextDecoderLayer(nn.Module): **kwargs: object, ): full_num_tokens = positions.shape[-1] - input_is_sequence_parallel = ( - self.use_attn_reduce_scatter_for_moe - and residual is not None - and hidden_states.shape[0] != full_num_tokens - ) if residual is None: residual = hidden_states @@ -493,7 +497,7 @@ class Qwen3NextDecoderLayer(nn.Module): else: hidden_states, residual = self.input_layernorm(hidden_states, residual) - if input_is_sequence_parallel: + if self.use_attn_reduce_scatter_for_moe: hidden_states = tensor_model_parallel_all_gather(hidden_states, 0) hidden_states = hidden_states[:full_num_tokens] @@ -524,8 +528,6 @@ class Qwen3NextDecoderLayer(nn.Module): # pad if not divisible by world size hidden_states = torch.nn.functional.pad(hidden_states, (0, 0, 0, sp_pad)) hidden_states = tensor_model_parallel_reduce_scatter(hidden_states, 0) - if not input_is_sequence_parallel: - residual = sequence_parallel_chunk(residual) # Fully Connected hidden_states, residual = self.post_attention_layernorm(hidden_states, residual) @@ -554,24 +556,6 @@ class Qwen3NextDecoderLayer(nn.Module): return hidden_states, residual -def _all_gather_hidden_and_residual( - hidden_states: torch.Tensor, - residual: torch.Tensor | None, - full_num_tokens: int, - hidden_size: int, -) -> tuple[torch.Tensor, torch.Tensor | None]: - if residual is None: - hidden_states = tensor_model_parallel_all_gather(hidden_states, 0) - hidden_states = hidden_states[:full_num_tokens] - return hidden_states, None - - combined_states = torch.cat([hidden_states, residual], dim=-1) - combined_states = tensor_model_parallel_all_gather(combined_states, 0) - combined_states = combined_states[:full_num_tokens] - hidden_states, residual = combined_states.split([hidden_size, hidden_size], dim=-1) - return hidden_states, residual - - @support_torch_compile class Qwen3NextModel(nn.Module, EagleModelMixin): hf_to_vllm_mapper = WeightsMapper( @@ -629,6 +613,10 @@ class Qwen3NextModel(nn.Module, EagleModelMixin): def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.embed_tokens(input_ids) + @property + def use_sequence_parallel(self) -> bool: + return self.layers[self.start_layer].use_attn_reduce_scatter_for_moe + def forward( self, input_ids: torch.Tensor | None, @@ -648,35 +636,20 @@ class Qwen3NextModel(nn.Module, EagleModelMixin): residual = intermediate_tensors["residual"] full_num_tokens = positions.shape[-1] + if self.use_sequence_parallel: + hidden_states = sequence_parallel_chunk(hidden_states) + assert residual is None + aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual) for layer_idx, layer in enumerate( islice(self.layers, self.start_layer, self.end_layer), start=self.start_layer, ): - if ( - hidden_states.shape[0] != full_num_tokens - and not layer.use_attn_reduce_scatter_for_moe - ): - hidden_states, residual = _all_gather_hidden_and_residual( - hidden_states, - residual, - full_num_tokens, - self.config.hidden_size, - ) hidden_states, residual = layer( positions=positions, hidden_states=hidden_states, residual=residual, ) - if (layer_idx + 1) in self.aux_hidden_state_layers and hidden_states.shape[ - 0 - ] != full_num_tokens: - hidden_states, residual = _all_gather_hidden_and_residual( - hidden_states, - residual, - full_num_tokens, - self.config.hidden_size, - ) self._maybe_add_hidden_state( aux_hidden_states, layer_idx + 1, hidden_states, residual ) @@ -685,14 +658,19 @@ class Qwen3NextModel(nn.Module, EagleModelMixin): return IntermediateTensors( {"hidden_states": hidden_states, "residual": residual} ) - if hidden_states.shape[0] != full_num_tokens: - hidden_states, residual = _all_gather_hidden_and_residual( - hidden_states, - residual, - full_num_tokens, - self.config.hidden_size, - ) hidden_states, _ = self.norm(hidden_states, residual) + if self.use_sequence_parallel: + if aux_hidden_states: + hidden_size = hidden_states.shape[-1] + hidden_states = torch.cat([hidden_states, *aux_hidden_states], dim=-1) + hidden_states = tensor_model_parallel_all_gather(hidden_states, 0) + hidden_states = hidden_states[:full_num_tokens] + hidden_states, *aux_hidden_states = hidden_states.split( + hidden_size, dim=-1 + ) + else: + hidden_states = tensor_model_parallel_all_gather(hidden_states, 0) + hidden_states = hidden_states[:full_num_tokens] if aux_hidden_states: return hidden_states, aux_hidden_states return hidden_states diff --git a/vllm/model_executor/models/qwen3_next_mtp.py b/vllm/model_executor/models/qwen3_next_mtp.py index c832e955b2..35f550700a 100644 --- a/vllm/model_executor/models/qwen3_next_mtp.py +++ b/vllm/model_executor/models/qwen3_next_mtp.py @@ -9,7 +9,7 @@ from torch import nn from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig -from vllm.distributed.parallel_state import get_pp_group +from vllm.distributed import get_pp_group, tensor_model_parallel_all_gather from vllm.logger import init_logger from vllm.model_executor.layers.linear import ColumnParallelLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor @@ -22,8 +22,8 @@ from vllm.model_executor.models.qwen3_next import ( Qwen3NextModel, Qwen3NextRMSNorm, QwenNextMixtureOfExperts, - _all_gather_hidden_and_residual, ) +from vllm.model_executor.models.utils import sequence_parallel_chunk from vllm.sequence import IntermediateTensors from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig @@ -131,6 +131,10 @@ class Qwen3NextMultiTokenPredictor(nn.Module): current_step_idx = spec_step_idx % self.num_mtp_layers mtp_layer = self.layers[current_step_idx] + if mtp_layer.use_attn_reduce_scatter_for_moe: + assert hidden_states.shape[0] == positions.shape[-1] + hidden_states = sequence_parallel_chunk(hidden_states) + assert residual is None hidden_states, residual = mtp_layer( positions=positions, hidden_states=hidden_states, @@ -142,14 +146,10 @@ class Qwen3NextMultiTokenPredictor(nn.Module): {"hidden_states": hidden_states, "residual": residual} ) - if mtp_layer.use_attn_reduce_scatter_for_moe: - hidden_states, residual = _all_gather_hidden_and_residual( - hidden_states, - residual, - positions.shape[-1], - self.config.hidden_size, - ) hidden_states, _ = self.norm(hidden_states, residual) + if mtp_layer.use_attn_reduce_scatter_for_moe: + hidden_states = tensor_model_parallel_all_gather(hidden_states, 0) + hidden_states = hidden_states[: positions.shape[-1]] return hidden_states def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: