[Bugfix] Fix SP pass for multimodal models and PP+SP residual handling (#33322)

Signed-off-by: Xingran Wang <wangxingran123456@outlook.com>
Signed-off-by: Hongjian Zhang <hirokenovo@gmail.com>
Co-authored-by: Hongjian Zhang <hirokenovo@gmail.com>
This commit is contained in:
Wang Xingran
2026-05-09 19:44:16 -07:00
committed by GitHub
co-authored by Hongjian Zhang
parent dcb3135af7
commit 0b272a6e01
3 changed files with 116 additions and 34 deletions
@@ -170,6 +170,7 @@ def _compare_sp(
num_gpus_available: int,
use_inductor_graph_partition: bool,
fuse_gemm_comms: bool,
enable_prompt_embeds: bool,
*,
method: Literal["generate", "encode"],
is_multimodal: bool,
@@ -253,6 +254,8 @@ def _compare_sp(
"--enable-mm-embeds",
]
)
elif enable_prompt_embeds:
common_args.append("--enable-prompt-embeds")
compilation_config = {
"mode": CompilationMode.VLLM_COMPILE,
@@ -262,7 +265,9 @@ def _compare_sp(
"fuse_gemm_comms": fuse_gemm_comms,
"fuse_norm_quant": fuse_norm_quant,
"fuse_act_quant": fuse_act_quant,
"fuse_allreduce_rms": False,
"eliminate_noops": True,
"sp_min_token_num": 0,
},
"use_inductor_graph_partition": use_inductor_graph_partition,
}
@@ -354,6 +359,49 @@ def test_tp_sp_generation(
num_gpus_available,
use_inductor_graph_partition,
fuse_gemm_comms=fuse_gemm_comms,
enable_prompt_embeds=False,
method="generate",
is_multimodal=False,
)
# Focused regression test for the SP + prompt_embeds graph-rewrite path.
# Covers pp_size=1 (SP only) and pp_size=2 (SP + PP); kept small on purpose so
# we don't double the matrix of `test_tp_sp_generation` above.
SP_PROMPT_EMBEDS_PARALLEL_SETUPS = [
ParallelSetup(
tp_size=2,
pp_size=pp_size,
fuse_norm_quant=False,
fuse_act_quant=False,
eager_mode=False,
chunked_prefill=False,
)
for pp_size in [1, 2]
]
@pytest.mark.parametrize("parallel_setup", SP_PROMPT_EMBEDS_PARALLEL_SETUPS)
@pytest.mark.parametrize("use_inductor_graph_partition", [True, False])
@create_new_process_for_each_test()
def test_tp_sp_generation_prompt_embeds(
parallel_setup: ParallelSetup,
num_gpus_available,
use_inductor_graph_partition: bool,
):
if use_inductor_graph_partition and not is_torch_equal_or_newer("2.9.0.dev"):
pytest.skip("inductor graph partition is only available in PyTorch 2.9+")
_compare_sp(
"hmellor/tiny-random-LlamaForCausalLM",
parallel_setup,
distributed_backend="mp",
runner="auto",
test_options=SPTestOptions(multi_node_only=False, load_format=None),
num_gpus_available=num_gpus_available,
use_inductor_graph_partition=use_inductor_graph_partition,
fuse_gemm_comms=False,
enable_prompt_embeds=True,
method="generate",
is_multimodal=False,
)