[Perf][Qwen] Replace MOE all-reduce with reduce-scatter (#47006)
Signed-off-by: gcanlin <canlinguosdu@gmail.com> Signed-off-by: yewentao256 <zhyanwentao@126.com> Co-authored-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
co-authored by
yewentao256
parent
fc1c548093
commit
a02984ed47
@@ -437,6 +437,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
gqa_interleaved_layout=False,
|
||||
reduce_results: bool = True,
|
||||
) -> None:
|
||||
super().__init__(config, vllm_config, prefix)
|
||||
|
||||
@@ -547,6 +548,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
input_is_parallel=True,
|
||||
reduce_results=reduce_results,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.out_proj",
|
||||
)
|
||||
|
||||
@@ -121,10 +121,17 @@ class Qwen3_5DecoderLayer(Qwen3NextDecoderLayer):
|
||||
config = vllm_config.model_config.hf_text_config
|
||||
model_config = vllm_config.model_config
|
||||
cache_config = vllm_config.cache_config
|
||||
parallel_config = vllm_config.parallel_config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
self.layer_type = layer_type
|
||||
self.layer_idx = extract_layer_index(prefix)
|
||||
is_moe_layer = config.model_type == "qwen3_5_moe_text"
|
||||
self.use_attn_reduce_scatter_for_moe = (
|
||||
parallel_config.use_sequence_parallel_moe
|
||||
and parallel_config.pipeline_parallel_size == 1
|
||||
and is_moe_layer
|
||||
)
|
||||
|
||||
if self.layer_type == "linear_attention":
|
||||
self.linear_attn = QwenGatedDeltaNetAttention(
|
||||
@@ -132,6 +139,7 @@ class Qwen3_5DecoderLayer(Qwen3NextDecoderLayer):
|
||||
vllm_config=vllm_config,
|
||||
prefix=f"{prefix}.linear_attn",
|
||||
gqa_interleaved_layout=False,
|
||||
reduce_results=not self.use_attn_reduce_scatter_for_moe,
|
||||
)
|
||||
elif self.layer_type == "full_attention":
|
||||
self.self_attn = Qwen3NextAttention(
|
||||
@@ -140,6 +148,7 @@ class Qwen3_5DecoderLayer(Qwen3NextDecoderLayer):
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
reduce_results=not self.use_attn_reduce_scatter_for_moe,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Invalid layer_type {self.layer_type}")
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm.distributed import (
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_world_size,
|
||||
tensor_model_parallel_all_gather,
|
||||
tensor_model_parallel_reduce_scatter,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
@@ -192,13 +193,17 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
else None,
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
already_sequence_parallel: bool = False,
|
||||
) -> torch.Tensor:
|
||||
# NOTE: hidden_states can have either 1D or 2D shape.
|
||||
orig_shape = hidden_states.shape
|
||||
num_tokens, hidden_dim = hidden_states.shape
|
||||
hidden_states = hidden_states.view(-1, hidden_dim)
|
||||
|
||||
if self.is_sequence_parallel:
|
||||
if self.is_sequence_parallel and not already_sequence_parallel:
|
||||
hidden_states = sequence_parallel_chunk(hidden_states)
|
||||
|
||||
if self.experts.is_internal_router:
|
||||
@@ -213,7 +218,7 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
hidden_states=hidden_states, router_logits=router_logits
|
||||
)
|
||||
|
||||
if self.is_sequence_parallel:
|
||||
if self.is_sequence_parallel and not already_sequence_parallel:
|
||||
final_hidden_states = tensor_model_parallel_all_gather(
|
||||
final_hidden_states, 0
|
||||
)
|
||||
@@ -229,6 +234,7 @@ class Qwen3NextAttention(nn.Module):
|
||||
model_config: ModelConfig | None = None,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
reduce_results: bool = True,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
@@ -271,6 +277,7 @@ class Qwen3NextAttention(nn.Module):
|
||||
self.total_num_heads * self.head_dim,
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
reduce_results=reduce_results,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
@@ -405,16 +412,31 @@ class Qwen3NextDecoderLayer(nn.Module):
|
||||
model_config = vllm_config.model_config
|
||||
cache_config = vllm_config.cache_config
|
||||
quant_config = vllm_config.quant_config
|
||||
parallel_config = vllm_config.parallel_config
|
||||
|
||||
self.layer_type = layer_type
|
||||
self.layer_idx = extract_layer_index(prefix)
|
||||
|
||||
mlp_only_layers = (
|
||||
[] if not hasattr(config, "mlp_only_layers") else config.mlp_only_layers
|
||||
)
|
||||
is_moe_layer = (self.layer_idx not in mlp_only_layers) and (
|
||||
config.num_experts > 0
|
||||
and (self.layer_idx + 1) % config.decoder_sparse_step == 0
|
||||
)
|
||||
self.use_attn_reduce_scatter_for_moe = (
|
||||
parallel_config.use_sequence_parallel_moe
|
||||
and parallel_config.pipeline_parallel_size == 1
|
||||
and is_moe_layer
|
||||
)
|
||||
|
||||
if self.layer_type == "linear_attention":
|
||||
self.linear_attn = QwenGatedDeltaNetAttention(
|
||||
config,
|
||||
vllm_config=vllm_config,
|
||||
prefix=f"{prefix}.linear_attn",
|
||||
gqa_interleaved_layout=True,
|
||||
reduce_results=not self.use_attn_reduce_scatter_for_moe,
|
||||
)
|
||||
elif self.layer_type == "full_attention":
|
||||
self.self_attn = Qwen3NextAttention(
|
||||
@@ -422,18 +444,13 @@ class Qwen3NextDecoderLayer(nn.Module):
|
||||
model_config=model_config,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
reduce_results=not self.use_attn_reduce_scatter_for_moe,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Invalid layer_type {self.layer_type}")
|
||||
|
||||
mlp_only_layers = (
|
||||
[] if not hasattr(config, "mlp_only_layers") else config.mlp_only_layers
|
||||
)
|
||||
if (self.layer_idx not in mlp_only_layers) and (
|
||||
config.num_experts > 0
|
||||
and (self.layer_idx + 1) % config.decoder_sparse_step == 0
|
||||
):
|
||||
if is_moe_layer:
|
||||
self.mlp = Qwen3NextSparseMoeBlock(
|
||||
vllm_config=vllm_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
@@ -478,12 +495,23 @@ class Qwen3NextDecoderLayer(nn.Module):
|
||||
positions: torch.Tensor = None,
|
||||
**kwargs: object,
|
||||
):
|
||||
full_num_tokens = positions.shape[-1]
|
||||
input_is_sequence_parallel = (
|
||||
self.use_attn_reduce_scatter_for_moe
|
||||
and residual is not None
|
||||
and hidden_states.shape[0] != full_num_tokens
|
||||
)
|
||||
|
||||
if residual is None:
|
||||
residual = hidden_states
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
else:
|
||||
hidden_states, residual = self.input_layernorm(hidden_states, residual)
|
||||
|
||||
if input_is_sequence_parallel:
|
||||
hidden_states = tensor_model_parallel_all_gather(hidden_states, 0)
|
||||
hidden_states = hidden_states[:full_num_tokens]
|
||||
|
||||
if self.layer_type == "linear_attention":
|
||||
hidden_states = self.linear_attn(hidden_states=hidden_states)
|
||||
elif self.layer_type == "full_attention":
|
||||
@@ -504,9 +532,25 @@ class Qwen3NextDecoderLayer(nn.Module):
|
||||
self.attn_layer_scale.to(hidden_states.dtype) + 1
|
||||
)
|
||||
|
||||
if self.use_attn_reduce_scatter_for_moe:
|
||||
tp_world_size = get_tensor_model_parallel_world_size()
|
||||
# small trick using minus, eg. -17 % 8 = 7
|
||||
sp_pad = (-hidden_states.shape[0]) % tp_world_size
|
||||
# pad if not divisible by world size
|
||||
hidden_states = torch.nn.functional.pad(hidden_states, (0, 0, 0, sp_pad))
|
||||
hidden_states = tensor_model_parallel_reduce_scatter(hidden_states, 0)
|
||||
if not input_is_sequence_parallel:
|
||||
residual = sequence_parallel_chunk(residual)
|
||||
|
||||
# Fully Connected
|
||||
hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
if self.use_attn_reduce_scatter_for_moe:
|
||||
hidden_states = self.mlp(
|
||||
hidden_states,
|
||||
already_sequence_parallel=True,
|
||||
)
|
||||
else:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
|
||||
if self.layer_scale:
|
||||
if len(hidden_states.shape) == 2:
|
||||
@@ -525,6 +569,24 @@ class Qwen3NextDecoderLayer(nn.Module):
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
def _all_gather_hidden_and_residual(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
full_num_tokens: int,
|
||||
hidden_size: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
if residual is None:
|
||||
hidden_states = tensor_model_parallel_all_gather(hidden_states, 0)
|
||||
hidden_states = hidden_states[:full_num_tokens]
|
||||
return hidden_states, None
|
||||
|
||||
combined_states = torch.cat([hidden_states, residual], dim=-1)
|
||||
combined_states = tensor_model_parallel_all_gather(combined_states, 0)
|
||||
combined_states = combined_states[:full_num_tokens]
|
||||
hidden_states, residual = combined_states.split([hidden_size, hidden_size], dim=-1)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
class Qwen3NextModel(nn.Module, EagleModelMixin):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
@@ -577,6 +639,8 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
|
||||
else:
|
||||
self.norm = PPMissingLayer()
|
||||
|
||||
self.aux_hidden_state_layers: tuple[int, ...] = ()
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
@@ -598,16 +662,36 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
residual = intermediate_tensors["residual"]
|
||||
|
||||
full_num_tokens = positions.shape[-1]
|
||||
aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual)
|
||||
for layer_idx, layer in enumerate(
|
||||
islice(self.layers, self.start_layer, self.end_layer),
|
||||
start=self.start_layer,
|
||||
):
|
||||
if (
|
||||
hidden_states.shape[0] != full_num_tokens
|
||||
and not layer.use_attn_reduce_scatter_for_moe
|
||||
):
|
||||
hidden_states, residual = _all_gather_hidden_and_residual(
|
||||
hidden_states,
|
||||
residual,
|
||||
full_num_tokens,
|
||||
self.config.hidden_size,
|
||||
)
|
||||
hidden_states, residual = layer(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
)
|
||||
if (layer_idx + 1) in self.aux_hidden_state_layers and hidden_states.shape[
|
||||
0
|
||||
] != full_num_tokens:
|
||||
hidden_states, residual = _all_gather_hidden_and_residual(
|
||||
hidden_states,
|
||||
residual,
|
||||
full_num_tokens,
|
||||
self.config.hidden_size,
|
||||
)
|
||||
self._maybe_add_hidden_state(
|
||||
aux_hidden_states, layer_idx + 1, hidden_states, residual
|
||||
)
|
||||
@@ -616,6 +700,13 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
|
||||
return IntermediateTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
if hidden_states.shape[0] != full_num_tokens:
|
||||
hidden_states, residual = _all_gather_hidden_and_residual(
|
||||
hidden_states,
|
||||
residual,
|
||||
full_num_tokens,
|
||||
self.config.hidden_size,
|
||||
)
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
if aux_hidden_states:
|
||||
return hidden_states, aux_hidden_states
|
||||
|
||||
Reference in New Issue
Block a user