[K3 Perf] Combine multiple all gather together for SP, 1.5~3x kernel level performance improvement (#51070)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -1118,13 +1118,6 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
|
||||
residual = intermediate_tensors["residual"]
|
||||
assert hidden_states is not None
|
||||
|
||||
aux_hidden_states: list[torch.Tensor] = []
|
||||
if self.start_layer in self.aux_hidden_state_layers:
|
||||
if self.use_attn_res or residual is None:
|
||||
aux_hidden_states.append(hidden_states)
|
||||
else:
|
||||
aux_hidden_states.append(hidden_states + residual)
|
||||
|
||||
full_num_tokens = positions.shape[0]
|
||||
if self.use_sequence_parallel:
|
||||
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
|
||||
@@ -1135,6 +1128,14 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
|
||||
hidden_states = sp_shard(hidden_states)
|
||||
assert residual is None, "Currently, SP is not supported with PP"
|
||||
|
||||
# sharded aux hidden states when sp is enabled
|
||||
aux_hidden_states: list[torch.Tensor] = []
|
||||
if self.start_layer in self.aux_hidden_state_layers:
|
||||
if self.use_attn_res or residual is None:
|
||||
aux_hidden_states.append(hidden_states)
|
||||
else:
|
||||
aux_hidden_states.append(hidden_states + residual)
|
||||
|
||||
prefix_sum = None
|
||||
if self.use_attn_res:
|
||||
block_residual = hidden_states.new_empty(
|
||||
@@ -1166,11 +1167,6 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
|
||||
assert residual is not None
|
||||
aux_hidden_state = hidden_states + residual
|
||||
|
||||
if self.use_sequence_parallel:
|
||||
# Gather SP-sharded aux hidden states.
|
||||
# TODO: Optimize this.
|
||||
aux_hidden_state = sp_all_gather(aux_hidden_state)
|
||||
aux_hidden_state = aux_hidden_state[:full_num_tokens]
|
||||
aux_hidden_states.append(aux_hidden_state)
|
||||
|
||||
assert hidden_states is not None
|
||||
@@ -1203,9 +1199,19 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
|
||||
hidden_states = hidden_states + residual
|
||||
|
||||
if self.use_sequence_parallel:
|
||||
# Gather SP-sharded hidden states.
|
||||
hidden_states = sp_all_gather(hidden_states)
|
||||
hidden_states = hidden_states[:full_num_tokens]
|
||||
if aux_hidden_states:
|
||||
hidden_size = hidden_states.shape[-1]
|
||||
packed_hidden_states = torch.cat(
|
||||
[hidden_states, *aux_hidden_states], dim=-1
|
||||
)
|
||||
packed_hidden_states = sp_all_gather(packed_hidden_states)
|
||||
packed_hidden_states = packed_hidden_states[:full_num_tokens]
|
||||
hidden_states, *aux_hidden_states = packed_hidden_states.split(
|
||||
hidden_size, dim=-1
|
||||
)
|
||||
else:
|
||||
hidden_states = sp_all_gather(hidden_states)
|
||||
hidden_states = hidden_states[:full_num_tokens]
|
||||
|
||||
# NOTE: the final norm is applied in compute_logits instead of here, so
|
||||
# the MTP draft model receives the pre-norm hidden states.
|
||||
|
||||
Reference in New Issue
Block a user