[K3 Perf] Combine multiple all gather together for SP, 1.5~3x kernel level performance improvement (#51070)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-08-05 17:42:41 -04:00
committed by GitHub
parent 373fe8b83e
commit 877975897d
+21 -15
View File
@@ -1118,13 +1118,6 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
residual = intermediate_tensors["residual"]
assert hidden_states is not None
aux_hidden_states: list[torch.Tensor] = []
if self.start_layer in self.aux_hidden_state_layers:
if self.use_attn_res or residual is None:
aux_hidden_states.append(hidden_states)
else:
aux_hidden_states.append(hidden_states + residual)
full_num_tokens = positions.shape[0]
if self.use_sequence_parallel:
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
@@ -1135,6 +1128,14 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
hidden_states = sp_shard(hidden_states)
assert residual is None, "Currently, SP is not supported with PP"
# sharded aux hidden states when sp is enabled
aux_hidden_states: list[torch.Tensor] = []
if self.start_layer in self.aux_hidden_state_layers:
if self.use_attn_res or residual is None:
aux_hidden_states.append(hidden_states)
else:
aux_hidden_states.append(hidden_states + residual)
prefix_sum = None
if self.use_attn_res:
block_residual = hidden_states.new_empty(
@@ -1166,11 +1167,6 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
assert residual is not None
aux_hidden_state = hidden_states + residual
if self.use_sequence_parallel:
# Gather SP-sharded aux hidden states.
# TODO: Optimize this.
aux_hidden_state = sp_all_gather(aux_hidden_state)
aux_hidden_state = aux_hidden_state[:full_num_tokens]
aux_hidden_states.append(aux_hidden_state)
assert hidden_states is not None
@@ -1203,9 +1199,19 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
hidden_states = hidden_states + residual
if self.use_sequence_parallel:
# Gather SP-sharded hidden states.
hidden_states = sp_all_gather(hidden_states)
hidden_states = hidden_states[:full_num_tokens]
if aux_hidden_states:
hidden_size = hidden_states.shape[-1]
packed_hidden_states = torch.cat(
[hidden_states, *aux_hidden_states], dim=-1
)
packed_hidden_states = sp_all_gather(packed_hidden_states)
packed_hidden_states = packed_hidden_states[:full_num_tokens]
hidden_states, *aux_hidden_states = packed_hidden_states.split(
hidden_size, dim=-1
)
else:
hidden_states = sp_all_gather(hidden_states)
hidden_states = hidden_states[:full_num_tokens]
# NOTE: the final norm is applied in compute_logits instead of here, so
# the MTP draft model receives the pre-norm hidden states.