diff --git a/vllm/models/kimi_k3/nvidia/model.py b/vllm/models/kimi_k3/nvidia/model.py index a008d17cb6..edb6c9c7d1 100644 --- a/vllm/models/kimi_k3/nvidia/model.py +++ b/vllm/models/kimi_k3/nvidia/model.py @@ -1118,13 +1118,6 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant): residual = intermediate_tensors["residual"] assert hidden_states is not None - aux_hidden_states: list[torch.Tensor] = [] - if self.start_layer in self.aux_hidden_state_layers: - if self.use_attn_res or residual is None: - aux_hidden_states.append(hidden_states) - else: - aux_hidden_states.append(hidden_states + residual) - full_num_tokens = positions.shape[0] if self.use_sequence_parallel: if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available(): @@ -1135,6 +1128,14 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant): hidden_states = sp_shard(hidden_states) assert residual is None, "Currently, SP is not supported with PP" + # sharded aux hidden states when sp is enabled + aux_hidden_states: list[torch.Tensor] = [] + if self.start_layer in self.aux_hidden_state_layers: + if self.use_attn_res or residual is None: + aux_hidden_states.append(hidden_states) + else: + aux_hidden_states.append(hidden_states + residual) + prefix_sum = None if self.use_attn_res: block_residual = hidden_states.new_empty( @@ -1166,11 +1167,6 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant): assert residual is not None aux_hidden_state = hidden_states + residual - if self.use_sequence_parallel: - # Gather SP-sharded aux hidden states. - # TODO: Optimize this. - aux_hidden_state = sp_all_gather(aux_hidden_state) - aux_hidden_state = aux_hidden_state[:full_num_tokens] aux_hidden_states.append(aux_hidden_state) assert hidden_states is not None @@ -1203,9 +1199,19 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant): hidden_states = hidden_states + residual if self.use_sequence_parallel: - # Gather SP-sharded hidden states. - hidden_states = sp_all_gather(hidden_states) - hidden_states = hidden_states[:full_num_tokens] + if aux_hidden_states: + hidden_size = hidden_states.shape[-1] + packed_hidden_states = torch.cat( + [hidden_states, *aux_hidden_states], dim=-1 + ) + packed_hidden_states = sp_all_gather(packed_hidden_states) + packed_hidden_states = packed_hidden_states[:full_num_tokens] + hidden_states, *aux_hidden_states = packed_hidden_states.split( + hidden_size, dim=-1 + ) + else: + hidden_states = sp_all_gather(hidden_states) + hidden_states = hidden_states[:full_num_tokens] # NOTE: the final norm is applied in compute_logits instead of here, so # the MTP draft model receives the pre-norm hidden states.