[Bugfix] Fix SP pass for multimodal models and PP+SP residual handling (#33322)

Signed-off-by: Xingran Wang <wangxingran123456@outlook.com>
Signed-off-by: Hongjian Zhang <hirokenovo@gmail.com>
Co-authored-by: Hongjian Zhang <hirokenovo@gmail.com>
This commit is contained in:
Wang Xingran
2026-05-09 19:44:16 -07:00
committed by GitHub
co-authored by Hongjian Zhang
parent dcb3135af7
commit 0b272a6e01
3 changed files with 116 additions and 34 deletions
@@ -170,6 +170,7 @@ def _compare_sp(
num_gpus_available: int,
use_inductor_graph_partition: bool,
fuse_gemm_comms: bool,
enable_prompt_embeds: bool,
*,
method: Literal["generate", "encode"],
is_multimodal: bool,
@@ -253,6 +254,8 @@ def _compare_sp(
"--enable-mm-embeds",
]
)
elif enable_prompt_embeds:
common_args.append("--enable-prompt-embeds")
compilation_config = {
"mode": CompilationMode.VLLM_COMPILE,
@@ -262,7 +265,9 @@ def _compare_sp(
"fuse_gemm_comms": fuse_gemm_comms,
"fuse_norm_quant": fuse_norm_quant,
"fuse_act_quant": fuse_act_quant,
"fuse_allreduce_rms": False,
"eliminate_noops": True,
"sp_min_token_num": 0,
},
"use_inductor_graph_partition": use_inductor_graph_partition,
}
@@ -354,6 +359,49 @@ def test_tp_sp_generation(
num_gpus_available,
use_inductor_graph_partition,
fuse_gemm_comms=fuse_gemm_comms,
enable_prompt_embeds=False,
method="generate",
is_multimodal=False,
)
# Focused regression test for the SP + prompt_embeds graph-rewrite path.
# Covers pp_size=1 (SP only) and pp_size=2 (SP + PP); kept small on purpose so
# we don't double the matrix of `test_tp_sp_generation` above.
SP_PROMPT_EMBEDS_PARALLEL_SETUPS = [
ParallelSetup(
tp_size=2,
pp_size=pp_size,
fuse_norm_quant=False,
fuse_act_quant=False,
eager_mode=False,
chunked_prefill=False,
)
for pp_size in [1, 2]
]
@pytest.mark.parametrize("parallel_setup", SP_PROMPT_EMBEDS_PARALLEL_SETUPS)
@pytest.mark.parametrize("use_inductor_graph_partition", [True, False])
@create_new_process_for_each_test()
def test_tp_sp_generation_prompt_embeds(
parallel_setup: ParallelSetup,
num_gpus_available,
use_inductor_graph_partition: bool,
):
if use_inductor_graph_partition and not is_torch_equal_or_newer("2.9.0.dev"):
pytest.skip("inductor graph partition is only available in PyTorch 2.9+")
_compare_sp(
"hmellor/tiny-random-LlamaForCausalLM",
parallel_setup,
distributed_backend="mp",
runner="auto",
test_options=SPTestOptions(multi_node_only=False, load_format=None),
num_gpus_available=num_gpus_available,
use_inductor_graph_partition=use_inductor_graph_partition,
fuse_gemm_comms=False,
enable_prompt_embeds=True,
method="generate",
is_multimodal=False,
)
@@ -15,7 +15,10 @@ import vllm.ir.ops
from vllm.config import VllmConfig
from vllm.config.utils import Range
from vllm.distributed import get_tp_group, tensor_model_parallel_all_reduce
from vllm.distributed.parallel_state import get_tensor_model_parallel_world_size
from vllm.distributed.parallel_state import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8StaticTensorSym,
@@ -122,6 +125,7 @@ class _SequenceParallelPatternHelper:
self.device = device
self.tp_group = get_tp_group()
self.tp_size = get_tensor_model_parallel_world_size()
self.tp_rank = get_tensor_model_parallel_rank()
def _all_reduce(self, x: torch.Tensor) -> torch.Tensor:
return tensor_model_parallel_all_reduce(x)
@@ -209,17 +213,35 @@ class MiddleAllReduceRMSNormPattern(_SequenceParallelPatternHelper):
mm_1: torch.Tensor,
rms_norm_weights: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
# pattern matcher replaces from top-to-bottom,
# so residual is still the full size here.
# once the seqpar pattern with the previous rmsnorm is replaced
# The pattern matcher replaces from the end of the graph
# (last layer first). At the time each match is replaced,
# the preceding layer has NOT been replaced yet, so
# `residual` is still full-size and the slice below is
# correct. Once the preceding layer IS replaced, its
# residual output shrinks to [local_len, H], and this
# slice becomes semantically incorrect (e.g. for rank > 0,
# the indices would be out of bounds). However, since the
# symbolic output shape equals the input shape,
# NoOpEliminationPass (called at the end of
# SequenceParallelismPass.__call__) removes these slices
# before the graph is ever executed or compiled.
reduce_scatter = self._reduce_scatter(mm_1)
residual = residual[0 : reduce_scatter.size(0), ...]
local_len = reduce_scatter.size(0)
# when the preceding VocabParallelEmbedding is excluded
# from the FX graph (e.g., passing `inputs_embeds` directly in VLMs),
# the FirstAllReduceRMSNorm pattern is never matched. we must
# perform a proper TP-aware slice here. simply using `[0:local_len]`
# would incorrectly cause all ranks to process rank 0's chunk.
residual = residual[
self.tp_rank * local_len : self.tp_rank * local_len + local_len, ...
]
rmsnorm = vllm.ir.ops.fused_add_rms_norm(
reduce_scatter, residual, rms_norm_weights, self.epsilon
)
all_gather = self._all_gather(rmsnorm[0])
# shape of residual changes but that's fine,
# next node is already slicing it, now becomes a noop
# residual output is now [local_len, H]; the next layer's
# slice on it is semantically incorrect until
# NoOpEliminationPass removes it.
return all_gather, rmsnorm[1]
pm.register_replacement(
@@ -309,19 +331,29 @@ class MiddleAllReduceRMSNormStaticFP8Pattern(_SequenceParallelPatternHelper):
rms_norm_weights: torch.Tensor,
scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
# pattern matcher replaces from top-to-bottom,
# so residual is still the full size here.
# add a temporary slice which will become a noop
# once the seqpar pattern with the previous rmsnorm is replaced
# See MiddleAllReduceRMSNormPattern.replacement for a
# detailed explanation of the temporary slice below:
# it is correct when first inserted, becomes semantically
# incorrect after the preceding layer is replaced, and is
# removed by NoOpEliminationPass before the graph is compiled.
reduce_scatter = self._reduce_scatter(mm_1)
residual = residual[0 : reduce_scatter.size(0), ...]
local_len = reduce_scatter.size(0)
# when the preceding VocabParallelEmbedding is excluded
# from the FX graph (e.g., passing `inputs_embeds` directly in VLMs),
# the FirstAllReduceRMSNorm pattern is never matched. we must
# perform a proper TP-aware slice here. simply using `[0:local_len]`
# would incorrectly cause all ranks to process rank 0's chunk.
residual = residual[
self.tp_rank * local_len : self.tp_rank * local_len + local_len, ...
]
rms, residual_out = vllm.ir.ops.fused_add_rms_norm(
reduce_scatter, residual, rms_norm_weights, self.epsilon
)
quant, _ = self.quant_matcher(rms, scale)
all_gather = self._all_gather(quant)
# shape of residual changes but that's fine,
# next node is already slicing it, now becomes a noop
# residual output is now [local_len, H]; the next layer's
# slice on it is semantically incorrect until
# NoOpEliminationPass removes it.
return all_gather, residual_out
pm.register_replacement(
@@ -485,12 +517,15 @@ class SequenceParallelismPass(VllmPatternMatcherPass):
gets split across TP ranks, causing size mismatches at subgraph
boundaries.
This pass splits up the residual tensor across TP ranks and hence
divides its size. Because the pattern matcher starts at the end of
the graph, the replacement contains a slice that temporarily conforms
the input residual to the correct size. After all patterns have been
matched, we use a NoOpEliminationPass to clean up what have now
become no-op slices.
This pass splits up the residual tensor across TP ranks and hence divides
its size. The pattern matcher starts at the end of the graph (last layer
first), so when each replacement inserts a residual slice, the preceding
layer has not been replaced yet and the slice is correct. Once the
preceding layer IS replaced, its residual output shrinks and the slice
becomes semantically incorrect (out-of-bounds indices for rank > 0).
The graph is never executed in this intermediate state —
NoOpEliminationPass removes these slices based on symbolic shape equality
(input shape == output shape) before the graph is compiled.
"""
@enable_fake_mode
+13 -14
View File
@@ -3129,7 +3129,7 @@ class GPUModelRunner(
return tuple(tasks)
def sync_and_slice_intermediate_tensors(
def sync_and_gather_intermediate_tensors(
self,
num_tokens: int,
intermediate_tensors: IntermediateTensors | None,
@@ -3140,24 +3140,23 @@ class GPUModelRunner(
tp = self.vllm_config.parallel_config.tensor_parallel_size
is_rs = is_residual_scattered_for_sp(self.vllm_config, num_tokens)
# When sequence parallelism is enabled, the "residual" tensor is sharded
# across tensor parallel ranks, so each rank only needs its own slice.
# When sequence parallelism is enabled, the "residual" tensor is
# sharded across TP ranks. All-gather it here because downstream
# QKV + Attention needs the full residual before the SP split point.
if sync_self:
assert intermediate_tensors is not None
for k, v in intermediate_tensors.items():
is_scattered = k == "residual" and is_rs
copy_len = num_tokens // tp if is_scattered else num_tokens
self.intermediate_tensors[k][:copy_len].copy_(
v[:copy_len], non_blocking=True
if is_scattered:
local_len = num_tokens // tp
v = get_tp_group().all_gather(v[:local_len], dim=0)
self.intermediate_tensors[k][:num_tokens].copy_(
v[:num_tokens], non_blocking=True
)
return IntermediateTensors(
{
k: v[: num_tokens // tp]
if k == "residual" and is_rs
else v[:num_tokens]
for k, v in self.intermediate_tensors.items()
}
{k: v[:num_tokens] for k, v in self.intermediate_tensors.items()}
)
def eplb_step(self, is_dummy: bool = False, is_profile: bool = False) -> None:
@@ -3373,7 +3372,7 @@ class GPUModelRunner(
intermediate_tensors = None
else:
assert intermediate_tensors is not None
intermediate_tensors = self.sync_and_slice_intermediate_tensors(
intermediate_tensors = self.sync_and_gather_intermediate_tensors(
num_input_tokens, intermediate_tensors, True
)
@@ -5589,7 +5588,7 @@ class GPUModelRunner(
)
)
intermediate_tensors = self.sync_and_slice_intermediate_tensors(
intermediate_tensors = self.sync_and_gather_intermediate_tensors(
num_tokens_padded, None, False
)