[Bugfix] Fix SP pass for multimodal models and PP+SP residual handling (#33322)
Signed-off-by: Xingran Wang <wangxingran123456@outlook.com> Signed-off-by: Hongjian Zhang <hirokenovo@gmail.com> Co-authored-by: Hongjian Zhang <hirokenovo@gmail.com>
This commit is contained in:
co-authored by
Hongjian Zhang
parent
dcb3135af7
commit
0b272a6e01
@@ -170,6 +170,7 @@ def _compare_sp(
|
||||
num_gpus_available: int,
|
||||
use_inductor_graph_partition: bool,
|
||||
fuse_gemm_comms: bool,
|
||||
enable_prompt_embeds: bool,
|
||||
*,
|
||||
method: Literal["generate", "encode"],
|
||||
is_multimodal: bool,
|
||||
@@ -253,6 +254,8 @@ def _compare_sp(
|
||||
"--enable-mm-embeds",
|
||||
]
|
||||
)
|
||||
elif enable_prompt_embeds:
|
||||
common_args.append("--enable-prompt-embeds")
|
||||
|
||||
compilation_config = {
|
||||
"mode": CompilationMode.VLLM_COMPILE,
|
||||
@@ -262,7 +265,9 @@ def _compare_sp(
|
||||
"fuse_gemm_comms": fuse_gemm_comms,
|
||||
"fuse_norm_quant": fuse_norm_quant,
|
||||
"fuse_act_quant": fuse_act_quant,
|
||||
"fuse_allreduce_rms": False,
|
||||
"eliminate_noops": True,
|
||||
"sp_min_token_num": 0,
|
||||
},
|
||||
"use_inductor_graph_partition": use_inductor_graph_partition,
|
||||
}
|
||||
@@ -354,6 +359,49 @@ def test_tp_sp_generation(
|
||||
num_gpus_available,
|
||||
use_inductor_graph_partition,
|
||||
fuse_gemm_comms=fuse_gemm_comms,
|
||||
enable_prompt_embeds=False,
|
||||
method="generate",
|
||||
is_multimodal=False,
|
||||
)
|
||||
|
||||
|
||||
# Focused regression test for the SP + prompt_embeds graph-rewrite path.
|
||||
# Covers pp_size=1 (SP only) and pp_size=2 (SP + PP); kept small on purpose so
|
||||
# we don't double the matrix of `test_tp_sp_generation` above.
|
||||
SP_PROMPT_EMBEDS_PARALLEL_SETUPS = [
|
||||
ParallelSetup(
|
||||
tp_size=2,
|
||||
pp_size=pp_size,
|
||||
fuse_norm_quant=False,
|
||||
fuse_act_quant=False,
|
||||
eager_mode=False,
|
||||
chunked_prefill=False,
|
||||
)
|
||||
for pp_size in [1, 2]
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("parallel_setup", SP_PROMPT_EMBEDS_PARALLEL_SETUPS)
|
||||
@pytest.mark.parametrize("use_inductor_graph_partition", [True, False])
|
||||
@create_new_process_for_each_test()
|
||||
def test_tp_sp_generation_prompt_embeds(
|
||||
parallel_setup: ParallelSetup,
|
||||
num_gpus_available,
|
||||
use_inductor_graph_partition: bool,
|
||||
):
|
||||
if use_inductor_graph_partition and not is_torch_equal_or_newer("2.9.0.dev"):
|
||||
pytest.skip("inductor graph partition is only available in PyTorch 2.9+")
|
||||
|
||||
_compare_sp(
|
||||
"hmellor/tiny-random-LlamaForCausalLM",
|
||||
parallel_setup,
|
||||
distributed_backend="mp",
|
||||
runner="auto",
|
||||
test_options=SPTestOptions(multi_node_only=False, load_format=None),
|
||||
num_gpus_available=num_gpus_available,
|
||||
use_inductor_graph_partition=use_inductor_graph_partition,
|
||||
fuse_gemm_comms=False,
|
||||
enable_prompt_embeds=True,
|
||||
method="generate",
|
||||
is_multimodal=False,
|
||||
)
|
||||
|
||||
@@ -15,7 +15,10 @@ import vllm.ir.ops
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.utils import Range
|
||||
from vllm.distributed import get_tp_group, tensor_model_parallel_all_reduce
|
||||
from vllm.distributed.parallel_state import get_tensor_model_parallel_world_size
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8StaticTensorSym,
|
||||
@@ -122,6 +125,7 @@ class _SequenceParallelPatternHelper:
|
||||
self.device = device
|
||||
self.tp_group = get_tp_group()
|
||||
self.tp_size = get_tensor_model_parallel_world_size()
|
||||
self.tp_rank = get_tensor_model_parallel_rank()
|
||||
|
||||
def _all_reduce(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return tensor_model_parallel_all_reduce(x)
|
||||
@@ -209,17 +213,35 @@ class MiddleAllReduceRMSNormPattern(_SequenceParallelPatternHelper):
|
||||
mm_1: torch.Tensor,
|
||||
rms_norm_weights: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# pattern matcher replaces from top-to-bottom,
|
||||
# so residual is still the full size here.
|
||||
# once the seqpar pattern with the previous rmsnorm is replaced
|
||||
# The pattern matcher replaces from the end of the graph
|
||||
# (last layer first). At the time each match is replaced,
|
||||
# the preceding layer has NOT been replaced yet, so
|
||||
# `residual` is still full-size and the slice below is
|
||||
# correct. Once the preceding layer IS replaced, its
|
||||
# residual output shrinks to [local_len, H], and this
|
||||
# slice becomes semantically incorrect (e.g. for rank > 0,
|
||||
# the indices would be out of bounds). However, since the
|
||||
# symbolic output shape equals the input shape,
|
||||
# NoOpEliminationPass (called at the end of
|
||||
# SequenceParallelismPass.__call__) removes these slices
|
||||
# before the graph is ever executed or compiled.
|
||||
reduce_scatter = self._reduce_scatter(mm_1)
|
||||
residual = residual[0 : reduce_scatter.size(0), ...]
|
||||
local_len = reduce_scatter.size(0)
|
||||
# when the preceding VocabParallelEmbedding is excluded
|
||||
# from the FX graph (e.g., passing `inputs_embeds` directly in VLMs),
|
||||
# the FirstAllReduceRMSNorm pattern is never matched. we must
|
||||
# perform a proper TP-aware slice here. simply using `[0:local_len]`
|
||||
# would incorrectly cause all ranks to process rank 0's chunk.
|
||||
residual = residual[
|
||||
self.tp_rank * local_len : self.tp_rank * local_len + local_len, ...
|
||||
]
|
||||
rmsnorm = vllm.ir.ops.fused_add_rms_norm(
|
||||
reduce_scatter, residual, rms_norm_weights, self.epsilon
|
||||
)
|
||||
all_gather = self._all_gather(rmsnorm[0])
|
||||
# shape of residual changes but that's fine,
|
||||
# next node is already slicing it, now becomes a noop
|
||||
# residual output is now [local_len, H]; the next layer's
|
||||
# slice on it is semantically incorrect until
|
||||
# NoOpEliminationPass removes it.
|
||||
return all_gather, rmsnorm[1]
|
||||
|
||||
pm.register_replacement(
|
||||
@@ -309,19 +331,29 @@ class MiddleAllReduceRMSNormStaticFP8Pattern(_SequenceParallelPatternHelper):
|
||||
rms_norm_weights: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# pattern matcher replaces from top-to-bottom,
|
||||
# so residual is still the full size here.
|
||||
# add a temporary slice which will become a noop
|
||||
# once the seqpar pattern with the previous rmsnorm is replaced
|
||||
# See MiddleAllReduceRMSNormPattern.replacement for a
|
||||
# detailed explanation of the temporary slice below:
|
||||
# it is correct when first inserted, becomes semantically
|
||||
# incorrect after the preceding layer is replaced, and is
|
||||
# removed by NoOpEliminationPass before the graph is compiled.
|
||||
reduce_scatter = self._reduce_scatter(mm_1)
|
||||
residual = residual[0 : reduce_scatter.size(0), ...]
|
||||
local_len = reduce_scatter.size(0)
|
||||
# when the preceding VocabParallelEmbedding is excluded
|
||||
# from the FX graph (e.g., passing `inputs_embeds` directly in VLMs),
|
||||
# the FirstAllReduceRMSNorm pattern is never matched. we must
|
||||
# perform a proper TP-aware slice here. simply using `[0:local_len]`
|
||||
# would incorrectly cause all ranks to process rank 0's chunk.
|
||||
residual = residual[
|
||||
self.tp_rank * local_len : self.tp_rank * local_len + local_len, ...
|
||||
]
|
||||
rms, residual_out = vllm.ir.ops.fused_add_rms_norm(
|
||||
reduce_scatter, residual, rms_norm_weights, self.epsilon
|
||||
)
|
||||
quant, _ = self.quant_matcher(rms, scale)
|
||||
all_gather = self._all_gather(quant)
|
||||
# shape of residual changes but that's fine,
|
||||
# next node is already slicing it, now becomes a noop
|
||||
# residual output is now [local_len, H]; the next layer's
|
||||
# slice on it is semantically incorrect until
|
||||
# NoOpEliminationPass removes it.
|
||||
return all_gather, residual_out
|
||||
|
||||
pm.register_replacement(
|
||||
@@ -485,12 +517,15 @@ class SequenceParallelismPass(VllmPatternMatcherPass):
|
||||
gets split across TP ranks, causing size mismatches at subgraph
|
||||
boundaries.
|
||||
|
||||
This pass splits up the residual tensor across TP ranks and hence
|
||||
divides its size. Because the pattern matcher starts at the end of
|
||||
the graph, the replacement contains a slice that temporarily conforms
|
||||
the input residual to the correct size. After all patterns have been
|
||||
matched, we use a NoOpEliminationPass to clean up what have now
|
||||
become no-op slices.
|
||||
This pass splits up the residual tensor across TP ranks and hence divides
|
||||
its size. The pattern matcher starts at the end of the graph (last layer
|
||||
first), so when each replacement inserts a residual slice, the preceding
|
||||
layer has not been replaced yet and the slice is correct. Once the
|
||||
preceding layer IS replaced, its residual output shrinks and the slice
|
||||
becomes semantically incorrect (out-of-bounds indices for rank > 0).
|
||||
The graph is never executed in this intermediate state —
|
||||
NoOpEliminationPass removes these slices based on symbolic shape equality
|
||||
(input shape == output shape) before the graph is compiled.
|
||||
"""
|
||||
|
||||
@enable_fake_mode
|
||||
|
||||
@@ -3129,7 +3129,7 @@ class GPUModelRunner(
|
||||
|
||||
return tuple(tasks)
|
||||
|
||||
def sync_and_slice_intermediate_tensors(
|
||||
def sync_and_gather_intermediate_tensors(
|
||||
self,
|
||||
num_tokens: int,
|
||||
intermediate_tensors: IntermediateTensors | None,
|
||||
@@ -3140,24 +3140,23 @@ class GPUModelRunner(
|
||||
tp = self.vllm_config.parallel_config.tensor_parallel_size
|
||||
is_rs = is_residual_scattered_for_sp(self.vllm_config, num_tokens)
|
||||
|
||||
# When sequence parallelism is enabled, the "residual" tensor is sharded
|
||||
# across tensor parallel ranks, so each rank only needs its own slice.
|
||||
# When sequence parallelism is enabled, the "residual" tensor is
|
||||
# sharded across TP ranks. All-gather it here because downstream
|
||||
# QKV + Attention needs the full residual before the SP split point.
|
||||
if sync_self:
|
||||
assert intermediate_tensors is not None
|
||||
for k, v in intermediate_tensors.items():
|
||||
is_scattered = k == "residual" and is_rs
|
||||
copy_len = num_tokens // tp if is_scattered else num_tokens
|
||||
self.intermediate_tensors[k][:copy_len].copy_(
|
||||
v[:copy_len], non_blocking=True
|
||||
if is_scattered:
|
||||
local_len = num_tokens // tp
|
||||
v = get_tp_group().all_gather(v[:local_len], dim=0)
|
||||
|
||||
self.intermediate_tensors[k][:num_tokens].copy_(
|
||||
v[:num_tokens], non_blocking=True
|
||||
)
|
||||
|
||||
return IntermediateTensors(
|
||||
{
|
||||
k: v[: num_tokens // tp]
|
||||
if k == "residual" and is_rs
|
||||
else v[:num_tokens]
|
||||
for k, v in self.intermediate_tensors.items()
|
||||
}
|
||||
{k: v[:num_tokens] for k, v in self.intermediate_tensors.items()}
|
||||
)
|
||||
|
||||
def eplb_step(self, is_dummy: bool = False, is_profile: bool = False) -> None:
|
||||
@@ -3373,7 +3372,7 @@ class GPUModelRunner(
|
||||
intermediate_tensors = None
|
||||
else:
|
||||
assert intermediate_tensors is not None
|
||||
intermediate_tensors = self.sync_and_slice_intermediate_tensors(
|
||||
intermediate_tensors = self.sync_and_gather_intermediate_tensors(
|
||||
num_input_tokens, intermediate_tensors, True
|
||||
)
|
||||
|
||||
@@ -5589,7 +5588,7 @@ class GPUModelRunner(
|
||||
)
|
||||
)
|
||||
|
||||
intermediate_tensors = self.sync_and_slice_intermediate_tensors(
|
||||
intermediate_tensors = self.sync_and_gather_intermediate_tensors(
|
||||
num_tokens_padded, None, False
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user