[Bugfix] Fix sequence parallelism bug when enable pipeline parallelism (#24021)

Signed-off-by: cascade812 <cascade812@outlook.com>
This commit is contained in:
cascade
2025-09-16 04:32:32 +00:00
committed by GitHub
parent 759ef49b15
commit 17871983a2
6 changed files with 135 additions and 42 deletions
+33 -30
View File
@@ -88,6 +88,7 @@ from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
from vllm.v1.worker.kv_connector_model_runner_mixin import (
KVConnectorModelRunnerMixin)
from vllm.v1.worker.lora_model_runner_mixin import LoRAModelRunnerMixin
from vllm.v1.worker.utils import is_residual_scattered_for_sp
from .utils import (AttentionGroup, MultiModalBudget,
add_kv_sharing_layers_to_kv_cache_groups, bind_kv_cache,
@@ -1633,21 +1634,14 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
assert self.intermediate_tensors is not None
tp = self.vllm_config.parallel_config.tensor_parallel_size
enabled_sp = self.compilation_config.pass_config. \
enable_sequence_parallelism
if enabled_sp:
# When sequence parallelism is enabled, we always pad num_tokens
# to be a multiple of tensor_parallel_size (tp) earlier
assert num_tokens % tp == 0
is_residual_scattered = tp > 1 and enabled_sp \
and num_tokens % tp == 0
is_rs = is_residual_scattered_for_sp(self.vllm_config, num_tokens)
# When sequence parallelism is enabled, the "residual" tensor is sharded
# across tensor parallel ranks, so each rank only needs its own slice.
if sync_self:
assert intermediate_tensors is not None
for k, v in intermediate_tensors.items():
is_scattered = k == "residual" and is_residual_scattered
is_scattered = k == "residual" and is_rs
copy_len = num_tokens // tp if is_scattered else \
num_tokens
self.intermediate_tensors[k][:copy_len].copy_(
@@ -1655,8 +1649,8 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
return IntermediateTensors({
k:
v[:num_tokens // tp]
if k == "residual" and is_residual_scattered else v[:num_tokens]
v[:num_tokens //
tp] if k == "residual" and is_rs else v[:num_tokens]
for k, v in self.intermediate_tensors.items()
})
@@ -1741,6 +1735,25 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
pooler_output=pooler_output,
)
def _get_num_input_tokens(self, num_scheduled_tokens: int) -> int:
if (self.compilation_config.cudagraph_mode != CUDAGraphMode.NONE
and not envs.VLLM_DISABLE_PAD_FOR_CUDAGRAPH
and hasattr(self, "cudagraph_batch_sizes")
and self.cudagraph_batch_sizes
and num_scheduled_tokens <= self.cudagraph_batch_sizes[-1]):
# Use CUDA graphs.
# Add padding to the batch size.
return self.vllm_config.pad_for_cudagraph(num_scheduled_tokens)
# Eager mode.
# Pad tokens to multiple of tensor_parallel_size when
# enabled collective fusion for SP
tp_size = self.vllm_config.parallel_config.tensor_parallel_size
if (self.compilation_config.pass_config.enable_sequence_parallelism
and tp_size > 1):
return round_up(num_scheduled_tokens, tp_size)
return num_scheduled_tokens
def _preprocess(
self,
scheduler_output: "SchedulerOutput",
@@ -1750,24 +1763,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
Optional[IntermediateTensors], dict[str, Any]]:
num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
if (self.compilation_config.cudagraph_mode != CUDAGraphMode.NONE
and not envs.VLLM_DISABLE_PAD_FOR_CUDAGRAPH
and num_scheduled_tokens <= self.cudagraph_batch_sizes[-1]):
# Use CUDA graphs.
# Add padding to the batch size.
num_input_tokens = self.vllm_config.pad_for_cudagraph(
num_scheduled_tokens)
else:
# Eager mode.
# Pad tokens to multiple of tensor_parallel_size when
# enabled collective fusion for SP
tp_size = self.vllm_config.parallel_config.tensor_parallel_size
if self.compilation_config.pass_config. \
enable_sequence_parallelism and tp_size > 1:
num_input_tokens = round_up(num_scheduled_tokens, tp_size)
else:
num_input_tokens = num_scheduled_tokens
num_input_tokens = self._get_num_input_tokens(num_scheduled_tokens)
# Padding for DP
num_pad, num_tokens_across_dp = self.get_dp_padding(num_input_tokens)
num_input_tokens += num_pad
@@ -2108,8 +2104,15 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
assert not self.is_pooling_model
if not get_pp_group().is_last_rank:
all_gather_tensors = {
"residual":
not is_residual_scattered_for_sp(
self.vllm_config, num_input_tokens)
}
get_pp_group().send_tensor_dict(
hidden_states.tensors, all_gather_group=get_tp_group())
hidden_states.tensors,
all_gather_group=get_tp_group(),
all_gather_tensors=all_gather_tensors)
logits = None
else:
sample_hidden_states = hidden_states[logits_indices]