[Bugfix][Refactor] Keep Qwen3Next layer boundaries sequence parallel (#50685)
Signed-off-by: kunzh <RimeK20@outlook.com> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
co-authored by
OpenAI Codex
zjy0516
parent
1be3628367
commit
c05d75aaa9
@@ -43,7 +43,7 @@ from .qwen3_5 import (
|
||||
Qwen3_5RMSNorm,
|
||||
)
|
||||
from .qwen3_5_mtp import Qwen3_5MoeMTP
|
||||
from .qwen3_next import Qwen3NextAttention, _all_gather_hidden_and_residual
|
||||
from .qwen3_next import Qwen3NextAttention
|
||||
from .qwen3_vl import (
|
||||
Qwen3_VisionTransformer,
|
||||
Qwen3VLDummyInputsBuilder,
|
||||
@@ -233,11 +233,6 @@ class InternS2MobiusDecoderLayer(nn.Module):
|
||||
meta_mlp: nn.ModuleList,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
full_num_tokens = positions.shape[-1]
|
||||
input_is_sequence_parallel = (
|
||||
self.use_attn_reduce_scatter_for_moe
|
||||
and residual is not None
|
||||
and hidden_states.shape[0] != full_num_tokens
|
||||
)
|
||||
|
||||
if residual is None:
|
||||
residual = hidden_states
|
||||
@@ -245,7 +240,7 @@ class InternS2MobiusDecoderLayer(nn.Module):
|
||||
else:
|
||||
hidden_states, residual = self.input_layernorm(hidden_states, residual)
|
||||
|
||||
if input_is_sequence_parallel:
|
||||
if self.use_attn_reduce_scatter_for_moe:
|
||||
hidden_states = tensor_model_parallel_all_gather(hidden_states, 0)
|
||||
hidden_states = hidden_states[:full_num_tokens]
|
||||
|
||||
@@ -266,8 +261,6 @@ class InternS2MobiusDecoderLayer(nn.Module):
|
||||
sp_pad = (-hidden_states.shape[0]) % tp_world_size
|
||||
hidden_states = torch.nn.functional.pad(hidden_states, (0, 0, 0, sp_pad))
|
||||
hidden_states = tensor_model_parallel_reduce_scatter(hidden_states, 0)
|
||||
if not input_is_sequence_parallel:
|
||||
residual = sequence_parallel_chunk(residual)
|
||||
|
||||
hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
|
||||
block_idx = self.layer_idx % self.num_blocks
|
||||
@@ -343,6 +336,10 @@ class InternS2MobiusModel(nn.Module, EagleModelMixin):
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
@property
|
||||
def use_sequence_parallel(self) -> bool:
|
||||
return self.layers[self.start_layer].use_attn_reduce_scatter_for_moe
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
@@ -363,36 +360,21 @@ class InternS2MobiusModel(nn.Module, EagleModelMixin):
|
||||
residual = intermediate_tensors["residual"]
|
||||
|
||||
full_num_tokens = positions.shape[-1]
|
||||
if self.use_sequence_parallel:
|
||||
hidden_states = sequence_parallel_chunk(hidden_states)
|
||||
assert residual is None
|
||||
|
||||
aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual)
|
||||
for layer_idx, layer in enumerate(
|
||||
islice(self.layers, self.start_layer, self.end_layer),
|
||||
start=self.start_layer,
|
||||
):
|
||||
if (
|
||||
hidden_states.shape[0] != full_num_tokens
|
||||
and not layer.use_attn_reduce_scatter_for_moe
|
||||
):
|
||||
hidden_states, residual = _all_gather_hidden_and_residual(
|
||||
hidden_states,
|
||||
residual,
|
||||
full_num_tokens,
|
||||
self.config.hidden_size,
|
||||
)
|
||||
hidden_states, residual = layer(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
meta_mlp=self.meta_mlp,
|
||||
)
|
||||
if (layer_idx + 1) in self.aux_hidden_state_layers and hidden_states.shape[
|
||||
0
|
||||
] != full_num_tokens:
|
||||
hidden_states, residual = _all_gather_hidden_and_residual(
|
||||
hidden_states,
|
||||
residual,
|
||||
full_num_tokens,
|
||||
self.config.hidden_size,
|
||||
)
|
||||
self._maybe_add_hidden_state(
|
||||
aux_hidden_states,
|
||||
layer_idx + 1,
|
||||
@@ -404,14 +386,19 @@ class InternS2MobiusModel(nn.Module, EagleModelMixin):
|
||||
return IntermediateTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
if hidden_states.shape[0] != full_num_tokens:
|
||||
hidden_states, residual = _all_gather_hidden_and_residual(
|
||||
hidden_states,
|
||||
residual,
|
||||
full_num_tokens,
|
||||
self.config.hidden_size,
|
||||
)
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
if self.use_sequence_parallel:
|
||||
if aux_hidden_states:
|
||||
hidden_size = hidden_states.shape[-1]
|
||||
hidden_states = torch.cat([hidden_states, *aux_hidden_states], dim=-1)
|
||||
hidden_states = tensor_model_parallel_all_gather(hidden_states, 0)
|
||||
hidden_states = hidden_states[:full_num_tokens]
|
||||
hidden_states, *aux_hidden_states = hidden_states.split(
|
||||
hidden_size, dim=-1
|
||||
)
|
||||
else:
|
||||
hidden_states = tensor_model_parallel_all_gather(hidden_states, 0)
|
||||
hidden_states = hidden_states[:full_num_tokens]
|
||||
if aux_hidden_states:
|
||||
return hidden_states, aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
Reference in New Issue
Block a user