From 877975897d68f5e837e25d2f8ba80f5355adbe5b Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Wed, 5 Aug 2026 17:42:41 -0400 Subject: [PATCH] [K3 Perf] Combine multiple all gather together for SP, 1.5~3x kernel level performance improvement (#51070) Signed-off-by: yewentao256 --- vllm/models/kimi_k3/nvidia/model.py | 36 +++++++++++++++++------------ 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/vllm/models/kimi_k3/nvidia/model.py b/vllm/models/kimi_k3/nvidia/model.py index a008d17cb6..edb6c9c7d1 100644 --- a/vllm/models/kimi_k3/nvidia/model.py +++ b/vllm/models/kimi_k3/nvidia/model.py @@ -1118,13 +1118,6 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant): residual = intermediate_tensors["residual"] assert hidden_states is not None - aux_hidden_states: list[torch.Tensor] = [] - if self.start_layer in self.aux_hidden_state_layers: - if self.use_attn_res or residual is None: - aux_hidden_states.append(hidden_states) - else: - aux_hidden_states.append(hidden_states + residual) - full_num_tokens = positions.shape[0] if self.use_sequence_parallel: if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available(): @@ -1135,6 +1128,14 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant): hidden_states = sp_shard(hidden_states) assert residual is None, "Currently, SP is not supported with PP" + # sharded aux hidden states when sp is enabled + aux_hidden_states: list[torch.Tensor] = [] + if self.start_layer in self.aux_hidden_state_layers: + if self.use_attn_res or residual is None: + aux_hidden_states.append(hidden_states) + else: + aux_hidden_states.append(hidden_states + residual) + prefix_sum = None if self.use_attn_res: block_residual = hidden_states.new_empty( @@ -1166,11 +1167,6 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant): assert residual is not None aux_hidden_state = hidden_states + residual - if self.use_sequence_parallel: - # Gather SP-sharded aux hidden states. - # TODO: Optimize this. - aux_hidden_state = sp_all_gather(aux_hidden_state) - aux_hidden_state = aux_hidden_state[:full_num_tokens] aux_hidden_states.append(aux_hidden_state) assert hidden_states is not None @@ -1203,9 +1199,19 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant): hidden_states = hidden_states + residual if self.use_sequence_parallel: - # Gather SP-sharded hidden states. - hidden_states = sp_all_gather(hidden_states) - hidden_states = hidden_states[:full_num_tokens] + if aux_hidden_states: + hidden_size = hidden_states.shape[-1] + packed_hidden_states = torch.cat( + [hidden_states, *aux_hidden_states], dim=-1 + ) + packed_hidden_states = sp_all_gather(packed_hidden_states) + packed_hidden_states = packed_hidden_states[:full_num_tokens] + hidden_states, *aux_hidden_states = packed_hidden_states.split( + hidden_size, dim=-1 + ) + else: + hidden_states = sp_all_gather(hidden_states) + hidden_states = hidden_states[:full_num_tokens] # NOTE: the final norm is applied in compute_logits instead of here, so # the MTP draft model receives the pre-norm hidden states.