From 0b272a6e01ab49274ce392dc1684d52143d5fd8e Mon Sep 17 00:00:00 2001 From: Wang Xingran <72983099+wangxingran222@users.noreply.github.com> Date: Sun, 10 May 2026 10:44:16 +0800 Subject: [PATCH] [Bugfix] Fix SP pass for multimodal models and PP+SP residual handling (#33322) Signed-off-by: Xingran Wang Signed-off-by: Hongjian Zhang Co-authored-by: Hongjian Zhang --- .../correctness_e2e/test_sequence_parallel.py | 48 ++++++++++++ .../passes/fusion/sequence_parallelism.py | 75 ++++++++++++++----- vllm/v1/worker/gpu_model_runner.py | 27 ++++--- 3 files changed, 116 insertions(+), 34 deletions(-) diff --git a/tests/compile/correctness_e2e/test_sequence_parallel.py b/tests/compile/correctness_e2e/test_sequence_parallel.py index 56d18b5d0e..be2e476693 100644 --- a/tests/compile/correctness_e2e/test_sequence_parallel.py +++ b/tests/compile/correctness_e2e/test_sequence_parallel.py @@ -170,6 +170,7 @@ def _compare_sp( num_gpus_available: int, use_inductor_graph_partition: bool, fuse_gemm_comms: bool, + enable_prompt_embeds: bool, *, method: Literal["generate", "encode"], is_multimodal: bool, @@ -253,6 +254,8 @@ def _compare_sp( "--enable-mm-embeds", ] ) + elif enable_prompt_embeds: + common_args.append("--enable-prompt-embeds") compilation_config = { "mode": CompilationMode.VLLM_COMPILE, @@ -262,7 +265,9 @@ def _compare_sp( "fuse_gemm_comms": fuse_gemm_comms, "fuse_norm_quant": fuse_norm_quant, "fuse_act_quant": fuse_act_quant, + "fuse_allreduce_rms": False, "eliminate_noops": True, + "sp_min_token_num": 0, }, "use_inductor_graph_partition": use_inductor_graph_partition, } @@ -354,6 +359,49 @@ def test_tp_sp_generation( num_gpus_available, use_inductor_graph_partition, fuse_gemm_comms=fuse_gemm_comms, + enable_prompt_embeds=False, + method="generate", + is_multimodal=False, + ) + + +# Focused regression test for the SP + prompt_embeds graph-rewrite path. +# Covers pp_size=1 (SP only) and pp_size=2 (SP + PP); kept small on purpose so +# we don't double the matrix of `test_tp_sp_generation` above. +SP_PROMPT_EMBEDS_PARALLEL_SETUPS = [ + ParallelSetup( + tp_size=2, + pp_size=pp_size, + fuse_norm_quant=False, + fuse_act_quant=False, + eager_mode=False, + chunked_prefill=False, + ) + for pp_size in [1, 2] +] + + +@pytest.mark.parametrize("parallel_setup", SP_PROMPT_EMBEDS_PARALLEL_SETUPS) +@pytest.mark.parametrize("use_inductor_graph_partition", [True, False]) +@create_new_process_for_each_test() +def test_tp_sp_generation_prompt_embeds( + parallel_setup: ParallelSetup, + num_gpus_available, + use_inductor_graph_partition: bool, +): + if use_inductor_graph_partition and not is_torch_equal_or_newer("2.9.0.dev"): + pytest.skip("inductor graph partition is only available in PyTorch 2.9+") + + _compare_sp( + "hmellor/tiny-random-LlamaForCausalLM", + parallel_setup, + distributed_backend="mp", + runner="auto", + test_options=SPTestOptions(multi_node_only=False, load_format=None), + num_gpus_available=num_gpus_available, + use_inductor_graph_partition=use_inductor_graph_partition, + fuse_gemm_comms=False, + enable_prompt_embeds=True, method="generate", is_multimodal=False, ) diff --git a/vllm/compilation/passes/fusion/sequence_parallelism.py b/vllm/compilation/passes/fusion/sequence_parallelism.py index f5255526c7..8d0f40e2c7 100644 --- a/vllm/compilation/passes/fusion/sequence_parallelism.py +++ b/vllm/compilation/passes/fusion/sequence_parallelism.py @@ -15,7 +15,10 @@ import vllm.ir.ops from vllm.config import VllmConfig from vllm.config.utils import Range from vllm.distributed import get_tp_group, tensor_model_parallel_all_reduce -from vllm.distributed.parallel_state import get_tensor_model_parallel_world_size +from vllm.distributed.parallel_state import ( + get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, +) from vllm.logger import init_logger from vllm.model_executor.layers.quantization.utils.quant_utils import ( kFp8StaticTensorSym, @@ -122,6 +125,7 @@ class _SequenceParallelPatternHelper: self.device = device self.tp_group = get_tp_group() self.tp_size = get_tensor_model_parallel_world_size() + self.tp_rank = get_tensor_model_parallel_rank() def _all_reduce(self, x: torch.Tensor) -> torch.Tensor: return tensor_model_parallel_all_reduce(x) @@ -209,17 +213,35 @@ class MiddleAllReduceRMSNormPattern(_SequenceParallelPatternHelper): mm_1: torch.Tensor, rms_norm_weights: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: - # pattern matcher replaces from top-to-bottom, - # so residual is still the full size here. - # once the seqpar pattern with the previous rmsnorm is replaced + # The pattern matcher replaces from the end of the graph + # (last layer first). At the time each match is replaced, + # the preceding layer has NOT been replaced yet, so + # `residual` is still full-size and the slice below is + # correct. Once the preceding layer IS replaced, its + # residual output shrinks to [local_len, H], and this + # slice becomes semantically incorrect (e.g. for rank > 0, + # the indices would be out of bounds). However, since the + # symbolic output shape equals the input shape, + # NoOpEliminationPass (called at the end of + # SequenceParallelismPass.__call__) removes these slices + # before the graph is ever executed or compiled. reduce_scatter = self._reduce_scatter(mm_1) - residual = residual[0 : reduce_scatter.size(0), ...] + local_len = reduce_scatter.size(0) + # when the preceding VocabParallelEmbedding is excluded + # from the FX graph (e.g., passing `inputs_embeds` directly in VLMs), + # the FirstAllReduceRMSNorm pattern is never matched. we must + # perform a proper TP-aware slice here. simply using `[0:local_len]` + # would incorrectly cause all ranks to process rank 0's chunk. + residual = residual[ + self.tp_rank * local_len : self.tp_rank * local_len + local_len, ... + ] rmsnorm = vllm.ir.ops.fused_add_rms_norm( reduce_scatter, residual, rms_norm_weights, self.epsilon ) all_gather = self._all_gather(rmsnorm[0]) - # shape of residual changes but that's fine, - # next node is already slicing it, now becomes a noop + # residual output is now [local_len, H]; the next layer's + # slice on it is semantically incorrect until + # NoOpEliminationPass removes it. return all_gather, rmsnorm[1] pm.register_replacement( @@ -309,19 +331,29 @@ class MiddleAllReduceRMSNormStaticFP8Pattern(_SequenceParallelPatternHelper): rms_norm_weights: torch.Tensor, scale: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: - # pattern matcher replaces from top-to-bottom, - # so residual is still the full size here. - # add a temporary slice which will become a noop - # once the seqpar pattern with the previous rmsnorm is replaced + # See MiddleAllReduceRMSNormPattern.replacement for a + # detailed explanation of the temporary slice below: + # it is correct when first inserted, becomes semantically + # incorrect after the preceding layer is replaced, and is + # removed by NoOpEliminationPass before the graph is compiled. reduce_scatter = self._reduce_scatter(mm_1) - residual = residual[0 : reduce_scatter.size(0), ...] + local_len = reduce_scatter.size(0) + # when the preceding VocabParallelEmbedding is excluded + # from the FX graph (e.g., passing `inputs_embeds` directly in VLMs), + # the FirstAllReduceRMSNorm pattern is never matched. we must + # perform a proper TP-aware slice here. simply using `[0:local_len]` + # would incorrectly cause all ranks to process rank 0's chunk. + residual = residual[ + self.tp_rank * local_len : self.tp_rank * local_len + local_len, ... + ] rms, residual_out = vllm.ir.ops.fused_add_rms_norm( reduce_scatter, residual, rms_norm_weights, self.epsilon ) quant, _ = self.quant_matcher(rms, scale) all_gather = self._all_gather(quant) - # shape of residual changes but that's fine, - # next node is already slicing it, now becomes a noop + # residual output is now [local_len, H]; the next layer's + # slice on it is semantically incorrect until + # NoOpEliminationPass removes it. return all_gather, residual_out pm.register_replacement( @@ -485,12 +517,15 @@ class SequenceParallelismPass(VllmPatternMatcherPass): gets split across TP ranks, causing size mismatches at subgraph boundaries. - This pass splits up the residual tensor across TP ranks and hence - divides its size. Because the pattern matcher starts at the end of - the graph, the replacement contains a slice that temporarily conforms - the input residual to the correct size. After all patterns have been - matched, we use a NoOpEliminationPass to clean up what have now - become no-op slices. + This pass splits up the residual tensor across TP ranks and hence divides + its size. The pattern matcher starts at the end of the graph (last layer + first), so when each replacement inserts a residual slice, the preceding + layer has not been replaced yet and the slice is correct. Once the + preceding layer IS replaced, its residual output shrinks and the slice + becomes semantically incorrect (out-of-bounds indices for rank > 0). + The graph is never executed in this intermediate state — + NoOpEliminationPass removes these slices based on symbolic shape equality + (input shape == output shape) before the graph is compiled. """ @enable_fake_mode diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index bbd4bb35f6..98b2fbf120 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -3129,7 +3129,7 @@ class GPUModelRunner( return tuple(tasks) - def sync_and_slice_intermediate_tensors( + def sync_and_gather_intermediate_tensors( self, num_tokens: int, intermediate_tensors: IntermediateTensors | None, @@ -3140,24 +3140,23 @@ class GPUModelRunner( tp = self.vllm_config.parallel_config.tensor_parallel_size is_rs = is_residual_scattered_for_sp(self.vllm_config, num_tokens) - # When sequence parallelism is enabled, the "residual" tensor is sharded - # across tensor parallel ranks, so each rank only needs its own slice. + # When sequence parallelism is enabled, the "residual" tensor is + # sharded across TP ranks. All-gather it here because downstream + # QKV + Attention needs the full residual before the SP split point. if sync_self: assert intermediate_tensors is not None for k, v in intermediate_tensors.items(): is_scattered = k == "residual" and is_rs - copy_len = num_tokens // tp if is_scattered else num_tokens - self.intermediate_tensors[k][:copy_len].copy_( - v[:copy_len], non_blocking=True + if is_scattered: + local_len = num_tokens // tp + v = get_tp_group().all_gather(v[:local_len], dim=0) + + self.intermediate_tensors[k][:num_tokens].copy_( + v[:num_tokens], non_blocking=True ) return IntermediateTensors( - { - k: v[: num_tokens // tp] - if k == "residual" and is_rs - else v[:num_tokens] - for k, v in self.intermediate_tensors.items() - } + {k: v[:num_tokens] for k, v in self.intermediate_tensors.items()} ) def eplb_step(self, is_dummy: bool = False, is_profile: bool = False) -> None: @@ -3373,7 +3372,7 @@ class GPUModelRunner( intermediate_tensors = None else: assert intermediate_tensors is not None - intermediate_tensors = self.sync_and_slice_intermediate_tensors( + intermediate_tensors = self.sync_and_gather_intermediate_tensors( num_input_tokens, intermediate_tensors, True ) @@ -5589,7 +5588,7 @@ class GPUModelRunner( ) ) - intermediate_tensors = self.sync_and_slice_intermediate_tensors( + intermediate_tensors = self.sync_and_gather_intermediate_tensors( num_tokens_padded, None, False )