[Bugfix][Refactor] Keep Qwen3Next layer boundaries sequence parallel (#50685)

Signed-off-by: kunzh <RimeK20@outlook.com>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
kzwrime
2026-08-14 11:58:32 +08:00
committed by GitHub
co-authored by OpenAI Codex zjy0516
parent 1be3628367
commit c05d75aaa9
4 changed files with 75 additions and 111 deletions
+22 -35
View File
@@ -43,7 +43,7 @@ from .qwen3_5 import (
Qwen3_5RMSNorm,
)
from .qwen3_5_mtp import Qwen3_5MoeMTP
from .qwen3_next import Qwen3NextAttention, _all_gather_hidden_and_residual
from .qwen3_next import Qwen3NextAttention
from .qwen3_vl import (
Qwen3_VisionTransformer,
Qwen3VLDummyInputsBuilder,
@@ -233,11 +233,6 @@ class InternS2MobiusDecoderLayer(nn.Module):
meta_mlp: nn.ModuleList,
) -> tuple[torch.Tensor, torch.Tensor]:
full_num_tokens = positions.shape[-1]
input_is_sequence_parallel = (
self.use_attn_reduce_scatter_for_moe
and residual is not None
and hidden_states.shape[0] != full_num_tokens
)
if residual is None:
residual = hidden_states
@@ -245,7 +240,7 @@ class InternS2MobiusDecoderLayer(nn.Module):
else:
hidden_states, residual = self.input_layernorm(hidden_states, residual)
if input_is_sequence_parallel:
if self.use_attn_reduce_scatter_for_moe:
hidden_states = tensor_model_parallel_all_gather(hidden_states, 0)
hidden_states = hidden_states[:full_num_tokens]
@@ -266,8 +261,6 @@ class InternS2MobiusDecoderLayer(nn.Module):
sp_pad = (-hidden_states.shape[0]) % tp_world_size
hidden_states = torch.nn.functional.pad(hidden_states, (0, 0, 0, sp_pad))
hidden_states = tensor_model_parallel_reduce_scatter(hidden_states, 0)
if not input_is_sequence_parallel:
residual = sequence_parallel_chunk(residual)
hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
block_idx = self.layer_idx % self.num_blocks
@@ -343,6 +336,10 @@ class InternS2MobiusModel(nn.Module, EagleModelMixin):
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
@property
def use_sequence_parallel(self) -> bool:
return self.layers[self.start_layer].use_attn_reduce_scatter_for_moe
def forward(
self,
input_ids: torch.Tensor | None,
@@ -363,36 +360,21 @@ class InternS2MobiusModel(nn.Module, EagleModelMixin):
residual = intermediate_tensors["residual"]
full_num_tokens = positions.shape[-1]
if self.use_sequence_parallel:
hidden_states = sequence_parallel_chunk(hidden_states)
assert residual is None
aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual)
for layer_idx, layer in enumerate(
islice(self.layers, self.start_layer, self.end_layer),
start=self.start_layer,
):
if (
hidden_states.shape[0] != full_num_tokens
and not layer.use_attn_reduce_scatter_for_moe
):
hidden_states, residual = _all_gather_hidden_and_residual(
hidden_states,
residual,
full_num_tokens,
self.config.hidden_size,
)
hidden_states, residual = layer(
positions=positions,
hidden_states=hidden_states,
residual=residual,
meta_mlp=self.meta_mlp,
)
if (layer_idx + 1) in self.aux_hidden_state_layers and hidden_states.shape[
0
] != full_num_tokens:
hidden_states, residual = _all_gather_hidden_and_residual(
hidden_states,
residual,
full_num_tokens,
self.config.hidden_size,
)
self._maybe_add_hidden_state(
aux_hidden_states,
layer_idx + 1,
@@ -404,14 +386,19 @@ class InternS2MobiusModel(nn.Module, EagleModelMixin):
return IntermediateTensors(
{"hidden_states": hidden_states, "residual": residual}
)
if hidden_states.shape[0] != full_num_tokens:
hidden_states, residual = _all_gather_hidden_and_residual(
hidden_states,
residual,
full_num_tokens,
self.config.hidden_size,
)
hidden_states, _ = self.norm(hidden_states, residual)
if self.use_sequence_parallel:
if aux_hidden_states:
hidden_size = hidden_states.shape[-1]
hidden_states = torch.cat([hidden_states, *aux_hidden_states], dim=-1)
hidden_states = tensor_model_parallel_all_gather(hidden_states, 0)
hidden_states = hidden_states[:full_num_tokens]
hidden_states, *aux_hidden_states = hidden_states.split(
hidden_size, dim=-1
)
else:
hidden_states = tensor_model_parallel_all_gather(hidden_states, 0)
hidden_states = hidden_states[:full_num_tokens]
if aux_hidden_states:
return hidden_states, aux_hidden_states
return hidden_states