[Bugfix] Fix SP pass for multimodal models and PP+SP residual handling (#33322)
Signed-off-by: Xingran Wang <wangxingran123456@outlook.com> Signed-off-by: Hongjian Zhang <hirokenovo@gmail.com> Co-authored-by: Hongjian Zhang <hirokenovo@gmail.com>
This commit is contained in:
co-authored by
Hongjian Zhang
parent
dcb3135af7
commit
0b272a6e01
@@ -170,6 +170,7 @@ def _compare_sp(
|
||||
num_gpus_available: int,
|
||||
use_inductor_graph_partition: bool,
|
||||
fuse_gemm_comms: bool,
|
||||
enable_prompt_embeds: bool,
|
||||
*,
|
||||
method: Literal["generate", "encode"],
|
||||
is_multimodal: bool,
|
||||
@@ -253,6 +254,8 @@ def _compare_sp(
|
||||
"--enable-mm-embeds",
|
||||
]
|
||||
)
|
||||
elif enable_prompt_embeds:
|
||||
common_args.append("--enable-prompt-embeds")
|
||||
|
||||
compilation_config = {
|
||||
"mode": CompilationMode.VLLM_COMPILE,
|
||||
@@ -262,7 +265,9 @@ def _compare_sp(
|
||||
"fuse_gemm_comms": fuse_gemm_comms,
|
||||
"fuse_norm_quant": fuse_norm_quant,
|
||||
"fuse_act_quant": fuse_act_quant,
|
||||
"fuse_allreduce_rms": False,
|
||||
"eliminate_noops": True,
|
||||
"sp_min_token_num": 0,
|
||||
},
|
||||
"use_inductor_graph_partition": use_inductor_graph_partition,
|
||||
}
|
||||
@@ -354,6 +359,49 @@ def test_tp_sp_generation(
|
||||
num_gpus_available,
|
||||
use_inductor_graph_partition,
|
||||
fuse_gemm_comms=fuse_gemm_comms,
|
||||
enable_prompt_embeds=False,
|
||||
method="generate",
|
||||
is_multimodal=False,
|
||||
)
|
||||
|
||||
|
||||
# Focused regression test for the SP + prompt_embeds graph-rewrite path.
|
||||
# Covers pp_size=1 (SP only) and pp_size=2 (SP + PP); kept small on purpose so
|
||||
# we don't double the matrix of `test_tp_sp_generation` above.
|
||||
SP_PROMPT_EMBEDS_PARALLEL_SETUPS = [
|
||||
ParallelSetup(
|
||||
tp_size=2,
|
||||
pp_size=pp_size,
|
||||
fuse_norm_quant=False,
|
||||
fuse_act_quant=False,
|
||||
eager_mode=False,
|
||||
chunked_prefill=False,
|
||||
)
|
||||
for pp_size in [1, 2]
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("parallel_setup", SP_PROMPT_EMBEDS_PARALLEL_SETUPS)
|
||||
@pytest.mark.parametrize("use_inductor_graph_partition", [True, False])
|
||||
@create_new_process_for_each_test()
|
||||
def test_tp_sp_generation_prompt_embeds(
|
||||
parallel_setup: ParallelSetup,
|
||||
num_gpus_available,
|
||||
use_inductor_graph_partition: bool,
|
||||
):
|
||||
if use_inductor_graph_partition and not is_torch_equal_or_newer("2.9.0.dev"):
|
||||
pytest.skip("inductor graph partition is only available in PyTorch 2.9+")
|
||||
|
||||
_compare_sp(
|
||||
"hmellor/tiny-random-LlamaForCausalLM",
|
||||
parallel_setup,
|
||||
distributed_backend="mp",
|
||||
runner="auto",
|
||||
test_options=SPTestOptions(multi_node_only=False, load_format=None),
|
||||
num_gpus_available=num_gpus_available,
|
||||
use_inductor_graph_partition=use_inductor_graph_partition,
|
||||
fuse_gemm_comms=False,
|
||||
enable_prompt_embeds=True,
|
||||
method="generate",
|
||||
is_multimodal=False,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user