[Bugfix][Refactor] Keep Qwen3Next layer boundaries sequence parallel (#50685)

Signed-off-by: kunzh <RimeK20@outlook.com>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
kzwrime
2026-08-14 11:58:32 +08:00
committed by GitHub
co-authored by OpenAI Codex zjy0516
parent 1be3628367
commit c05d75aaa9
4 changed files with 75 additions and 111 deletions
+22 -35
View File
@@ -43,7 +43,7 @@ from .qwen3_5 import (
Qwen3_5RMSNorm,
)
from .qwen3_5_mtp import Qwen3_5MoeMTP
from .qwen3_next import Qwen3NextAttention, _all_gather_hidden_and_residual
from .qwen3_next import Qwen3NextAttention
from .qwen3_vl import (
Qwen3_VisionTransformer,
Qwen3VLDummyInputsBuilder,
@@ -233,11 +233,6 @@ class InternS2MobiusDecoderLayer(nn.Module):
meta_mlp: nn.ModuleList,
) -> tuple[torch.Tensor, torch.Tensor]:
full_num_tokens = positions.shape[-1]
input_is_sequence_parallel = (
self.use_attn_reduce_scatter_for_moe
and residual is not None
and hidden_states.shape[0] != full_num_tokens
)
if residual is None:
residual = hidden_states
@@ -245,7 +240,7 @@ class InternS2MobiusDecoderLayer(nn.Module):
else:
hidden_states, residual = self.input_layernorm(hidden_states, residual)
if input_is_sequence_parallel:
if self.use_attn_reduce_scatter_for_moe:
hidden_states = tensor_model_parallel_all_gather(hidden_states, 0)
hidden_states = hidden_states[:full_num_tokens]
@@ -266,8 +261,6 @@ class InternS2MobiusDecoderLayer(nn.Module):
sp_pad = (-hidden_states.shape[0]) % tp_world_size
hidden_states = torch.nn.functional.pad(hidden_states, (0, 0, 0, sp_pad))
hidden_states = tensor_model_parallel_reduce_scatter(hidden_states, 0)
if not input_is_sequence_parallel:
residual = sequence_parallel_chunk(residual)
hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
block_idx = self.layer_idx % self.num_blocks
@@ -343,6 +336,10 @@ class InternS2MobiusModel(nn.Module, EagleModelMixin):
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
@property
def use_sequence_parallel(self) -> bool:
return self.layers[self.start_layer].use_attn_reduce_scatter_for_moe
def forward(
self,
input_ids: torch.Tensor | None,
@@ -363,36 +360,21 @@ class InternS2MobiusModel(nn.Module, EagleModelMixin):
residual = intermediate_tensors["residual"]
full_num_tokens = positions.shape[-1]
if self.use_sequence_parallel:
hidden_states = sequence_parallel_chunk(hidden_states)
assert residual is None
aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual)
for layer_idx, layer in enumerate(
islice(self.layers, self.start_layer, self.end_layer),
start=self.start_layer,
):
if (
hidden_states.shape[0] != full_num_tokens
and not layer.use_attn_reduce_scatter_for_moe
):
hidden_states, residual = _all_gather_hidden_and_residual(
hidden_states,
residual,
full_num_tokens,
self.config.hidden_size,
)
hidden_states, residual = layer(
positions=positions,
hidden_states=hidden_states,
residual=residual,
meta_mlp=self.meta_mlp,
)
if (layer_idx + 1) in self.aux_hidden_state_layers and hidden_states.shape[
0
] != full_num_tokens:
hidden_states, residual = _all_gather_hidden_and_residual(
hidden_states,
residual,
full_num_tokens,
self.config.hidden_size,
)
self._maybe_add_hidden_state(
aux_hidden_states,
layer_idx + 1,
@@ -404,14 +386,19 @@ class InternS2MobiusModel(nn.Module, EagleModelMixin):
return IntermediateTensors(
{"hidden_states": hidden_states, "residual": residual}
)
if hidden_states.shape[0] != full_num_tokens:
hidden_states, residual = _all_gather_hidden_and_residual(
hidden_states,
residual,
full_num_tokens,
self.config.hidden_size,
)
hidden_states, _ = self.norm(hidden_states, residual)
if self.use_sequence_parallel:
if aux_hidden_states:
hidden_size = hidden_states.shape[-1]
hidden_states = torch.cat([hidden_states, *aux_hidden_states], dim=-1)
hidden_states = tensor_model_parallel_all_gather(hidden_states, 0)
hidden_states = hidden_states[:full_num_tokens]
hidden_states, *aux_hidden_states = hidden_states.split(
hidden_size, dim=-1
)
else:
hidden_states = tensor_model_parallel_all_gather(hidden_states, 0)
hidden_states = hidden_states[:full_num_tokens]
if aux_hidden_states:
return hidden_states, aux_hidden_states
return hidden_states
+9 -10
View File
@@ -10,7 +10,7 @@ from torch import nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.distributed.parallel_state import get_pp_group
from vllm.distributed import get_pp_group, tensor_model_parallel_all_gather
from vllm.logger import init_logger
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -26,9 +26,9 @@ from vllm.model_executor.models.qwen3_5 import (
)
from vllm.model_executor.models.qwen3_next import (
QwenNextMixtureOfExperts,
_all_gather_hidden_and_residual,
_is_shared_expert_fse_compatible,
)
from vllm.model_executor.models.utils import sequence_parallel_chunk
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.qwen3_5 import Qwen3_5TextConfig
from vllm.transformers_utils.configs.qwen3_5_moe import Qwen3_5MoeTextConfig
@@ -161,6 +161,10 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
current_step_idx = spec_step_idx % self.num_mtp_layers
mtp_layer = self.layers[current_step_idx]
if mtp_layer.use_attn_reduce_scatter_for_moe:
assert hidden_states.shape[0] == positions.shape[-1]
hidden_states = sequence_parallel_chunk(hidden_states)
assert residual is None
hidden_states, residual = mtp_layer(
positions=positions,
hidden_states=hidden_states,
@@ -172,15 +176,10 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
{"hidden_states": hidden_states, "residual": residual}
)
if mtp_layer.use_attn_reduce_scatter_for_moe:
hidden_states, residual = _all_gather_hidden_and_residual(
hidden_states,
residual,
positions.shape[-1],
self.config.hidden_size,
)
hidden_states, _ = self.norm(hidden_states, residual)
if mtp_layer.use_attn_reduce_scatter_for_moe:
hidden_states = tensor_model_parallel_all_gather(hidden_states, 0)
hidden_states = hidden_states[: positions.shape[-1]]
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+35 -57
View File
@@ -78,6 +78,18 @@ logger = init_logger(__name__)
KVCache = tuple[torch.Tensor, torch.Tensor]
def _should_use_sequence_parallel(vllm_config: VllmConfig) -> bool:
config = vllm_config.model_config.hf_text_config
parallel_config = vllm_config.parallel_config
return (
parallel_config.use_sequence_parallel_moe
and parallel_config.pipeline_parallel_size == 1
and getattr(config, "num_experts", 0) > 0
and not getattr(config, "mlp_only_layers", [])
and getattr(config, "decoder_sparse_step", 1) == 1
)
def _is_shared_expert_fse_compatible(quant_config) -> bool:
"""Check if shared expert can be fused with routed experts.
@@ -397,7 +409,6 @@ class Qwen3NextDecoderLayer(nn.Module):
model_config = vllm_config.model_config
cache_config = vllm_config.cache_config
quant_config = vllm_config.quant_config
parallel_config = vllm_config.parallel_config
self.layer_type = layer_type
self.layer_idx = extract_layer_index(prefix)
@@ -409,10 +420,8 @@ class Qwen3NextDecoderLayer(nn.Module):
config.num_experts > 0
and (self.layer_idx + 1) % config.decoder_sparse_step == 0
)
self.use_attn_reduce_scatter_for_moe = (
parallel_config.use_sequence_parallel_moe
and parallel_config.pipeline_parallel_size == 1
and is_moe_layer
self.use_attn_reduce_scatter_for_moe = _should_use_sequence_parallel(
vllm_config
)
if self.layer_type == "linear_attention":
@@ -481,11 +490,6 @@ class Qwen3NextDecoderLayer(nn.Module):
**kwargs: object,
):
full_num_tokens = positions.shape[-1]
input_is_sequence_parallel = (
self.use_attn_reduce_scatter_for_moe
and residual is not None
and hidden_states.shape[0] != full_num_tokens
)
if residual is None:
residual = hidden_states
@@ -493,7 +497,7 @@ class Qwen3NextDecoderLayer(nn.Module):
else:
hidden_states, residual = self.input_layernorm(hidden_states, residual)
if input_is_sequence_parallel:
if self.use_attn_reduce_scatter_for_moe:
hidden_states = tensor_model_parallel_all_gather(hidden_states, 0)
hidden_states = hidden_states[:full_num_tokens]
@@ -524,8 +528,6 @@ class Qwen3NextDecoderLayer(nn.Module):
# pad if not divisible by world size
hidden_states = torch.nn.functional.pad(hidden_states, (0, 0, 0, sp_pad))
hidden_states = tensor_model_parallel_reduce_scatter(hidden_states, 0)
if not input_is_sequence_parallel:
residual = sequence_parallel_chunk(residual)
# Fully Connected
hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
@@ -554,24 +556,6 @@ class Qwen3NextDecoderLayer(nn.Module):
return hidden_states, residual
def _all_gather_hidden_and_residual(
hidden_states: torch.Tensor,
residual: torch.Tensor | None,
full_num_tokens: int,
hidden_size: int,
) -> tuple[torch.Tensor, torch.Tensor | None]:
if residual is None:
hidden_states = tensor_model_parallel_all_gather(hidden_states, 0)
hidden_states = hidden_states[:full_num_tokens]
return hidden_states, None
combined_states = torch.cat([hidden_states, residual], dim=-1)
combined_states = tensor_model_parallel_all_gather(combined_states, 0)
combined_states = combined_states[:full_num_tokens]
hidden_states, residual = combined_states.split([hidden_size, hidden_size], dim=-1)
return hidden_states, residual
@support_torch_compile
class Qwen3NextModel(nn.Module, EagleModelMixin):
hf_to_vllm_mapper = WeightsMapper(
@@ -629,6 +613,10 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
@property
def use_sequence_parallel(self) -> bool:
return self.layers[self.start_layer].use_attn_reduce_scatter_for_moe
def forward(
self,
input_ids: torch.Tensor | None,
@@ -648,35 +636,20 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
residual = intermediate_tensors["residual"]
full_num_tokens = positions.shape[-1]
if self.use_sequence_parallel:
hidden_states = sequence_parallel_chunk(hidden_states)
assert residual is None
aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual)
for layer_idx, layer in enumerate(
islice(self.layers, self.start_layer, self.end_layer),
start=self.start_layer,
):
if (
hidden_states.shape[0] != full_num_tokens
and not layer.use_attn_reduce_scatter_for_moe
):
hidden_states, residual = _all_gather_hidden_and_residual(
hidden_states,
residual,
full_num_tokens,
self.config.hidden_size,
)
hidden_states, residual = layer(
positions=positions,
hidden_states=hidden_states,
residual=residual,
)
if (layer_idx + 1) in self.aux_hidden_state_layers and hidden_states.shape[
0
] != full_num_tokens:
hidden_states, residual = _all_gather_hidden_and_residual(
hidden_states,
residual,
full_num_tokens,
self.config.hidden_size,
)
self._maybe_add_hidden_state(
aux_hidden_states, layer_idx + 1, hidden_states, residual
)
@@ -685,14 +658,19 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
return IntermediateTensors(
{"hidden_states": hidden_states, "residual": residual}
)
if hidden_states.shape[0] != full_num_tokens:
hidden_states, residual = _all_gather_hidden_and_residual(
hidden_states,
residual,
full_num_tokens,
self.config.hidden_size,
)
hidden_states, _ = self.norm(hidden_states, residual)
if self.use_sequence_parallel:
if aux_hidden_states:
hidden_size = hidden_states.shape[-1]
hidden_states = torch.cat([hidden_states, *aux_hidden_states], dim=-1)
hidden_states = tensor_model_parallel_all_gather(hidden_states, 0)
hidden_states = hidden_states[:full_num_tokens]
hidden_states, *aux_hidden_states = hidden_states.split(
hidden_size, dim=-1
)
else:
hidden_states = tensor_model_parallel_all_gather(hidden_states, 0)
hidden_states = hidden_states[:full_num_tokens]
if aux_hidden_states:
return hidden_states, aux_hidden_states
return hidden_states
+9 -9
View File
@@ -9,7 +9,7 @@ from torch import nn
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.distributed.parallel_state import get_pp_group
from vllm.distributed import get_pp_group, tensor_model_parallel_all_gather
from vllm.logger import init_logger
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -22,8 +22,8 @@ from vllm.model_executor.models.qwen3_next import (
Qwen3NextModel,
Qwen3NextRMSNorm,
QwenNextMixtureOfExperts,
_all_gather_hidden_and_residual,
)
from vllm.model_executor.models.utils import sequence_parallel_chunk
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
@@ -131,6 +131,10 @@ class Qwen3NextMultiTokenPredictor(nn.Module):
current_step_idx = spec_step_idx % self.num_mtp_layers
mtp_layer = self.layers[current_step_idx]
if mtp_layer.use_attn_reduce_scatter_for_moe:
assert hidden_states.shape[0] == positions.shape[-1]
hidden_states = sequence_parallel_chunk(hidden_states)
assert residual is None
hidden_states, residual = mtp_layer(
positions=positions,
hidden_states=hidden_states,
@@ -142,14 +146,10 @@ class Qwen3NextMultiTokenPredictor(nn.Module):
{"hidden_states": hidden_states, "residual": residual}
)
if mtp_layer.use_attn_reduce_scatter_for_moe:
hidden_states, residual = _all_gather_hidden_and_residual(
hidden_states,
residual,
positions.shape[-1],
self.config.hidden_size,
)
hidden_states, _ = self.norm(hidden_states, residual)
if mtp_layer.use_attn_reduce_scatter_for_moe:
hidden_states = tensor_model_parallel_all_gather(hidden_states, 0)
hidden_states = hidden_states[: positions.shape[-1]]
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: