[torch.compile]: Disable Sequence Parallelism (SP) for piecewise compilation (#38373)

Signed-off-by: SouthWest7 <am1ao@qq.com>
Signed-off-by: Xinan Miao <1403572259@qq.com>
Co-authored-by: SouthWest7 <am1ao@qq.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Wang Xingran <72983099+wangxingran222@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Xinan Miao
2026-04-26 17:44:42 +00:00
committed by GitHub
co-authored by SouthWest7 gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> OpenAI Codex Wang Xingran mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent b39c266dae
commit 32e45636e3
9 changed files with 222 additions and 79 deletions
@@ -261,6 +261,8 @@ def _compare_sp(
},
"use_inductor_graph_partition": use_inductor_graph_partition,
}
if not use_inductor_graph_partition:
compilation_config["splitting_ops"] = []
tp_sp_args = [
*common_args,
@@ -19,6 +19,7 @@ from vllm.config import (
VllmConfig,
set_current_vllm_config,
)
from vllm.config.utils import Range
from vllm.distributed import (
tensor_model_parallel_all_gather,
tensor_model_parallel_reduce_scatter,
@@ -288,6 +289,22 @@ def test_async_tp_pass_replace(
run_torch_spawn(async_tp_pass_on_test_model, num_processes)
def test_async_tp_pass_requires_full_graph_compilation():
vllm_config = VllmConfig()
vllm_config.compilation_config.use_inductor_graph_partition = False
vllm_config.compilation_config.splitting_ops = [
"vllm::unified_attention_with_output"
]
async_tp_pass = object.__new__(AsyncTPPass)
async_tp_pass.compilation_config = vllm_config.compilation_config
with pytest.raises(
AssertionError, match="AsyncTPPass requires full-graph compilation"
):
async_tp_pass.is_applicable_for_range(Range(start=8, end=8))
def async_tp_pass_on_test_model(
local_rank: int,
world_size: int,
@@ -22,6 +22,7 @@ from vllm.config import (
get_current_vllm_config,
set_current_vllm_config,
)
from vllm.config.utils import Range
from vllm.distributed import tensor_model_parallel_all_reduce
from vllm.distributed.parallel_state import (
init_distributed_environment,
@@ -216,6 +217,24 @@ def test_sequence_parallelism_pass(
run_torch_spawn(sequence_parallelism_pass_on_test_model, num_processes)
def test_sequence_parallelism_pass_requires_full_graph_compilation():
vllm_config = VllmConfig()
vllm_config.compilation_config.use_inductor_graph_partition = False
vllm_config.compilation_config.splitting_ops = [
"vllm::unified_attention_with_output"
]
sequence_parallelism_pass = object.__new__(SequenceParallelismPass)
sequence_parallelism_pass.compilation_config = vllm_config.compilation_config
sequence_parallelism_pass.min_token_num = 1
with pytest.raises(
AssertionError,
match="SequenceParallelismPass requires full-graph compilation",
):
sequence_parallelism_pass.is_applicable_for_range(Range(start=8, end=8))
def sequence_parallelism_pass_on_test_model(
local_rank: int,
world_size: int,
+118 -1
View File
@@ -407,7 +407,7 @@ def test_should_split():
(None, 257, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 256),
# max from list
([1, 2, 4, 15], None, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 15),
# filtered out 15 due to SP
# SP forces full-graph compilation, sizes are filtered by TP
([1, 2, 4, 15], None, 2, True, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 4),
# limited by the max_tokens
([1, 2, 4, 15], None, 1, False, 8, CUDAGraphMode.FULL_AND_PIECEWISE, 4),
@@ -465,6 +465,123 @@ def test_cudagraph_sizes_post_init(
)
@pytest.mark.skipif(
not current_platform.support_static_graph_mode(),
reason="Skip if not cudagraph mode supported",
)
@pytest.mark.parametrize(
(
"cudagraph_mode",
"use_inductor_graph_partition",
"expected_enable_sp",
"expected_cudagraph_mode",
"expected_piecewise_compile",
"expected_capture_sizes",
"expected_max_size",
),
[
(CUDAGraphMode.PIECEWISE, False, True, CUDAGraphMode.FULL, False, [2, 4], 4),
(
CUDAGraphMode.FULL_DECODE_ONLY,
False,
True,
CUDAGraphMode.FULL_DECODE_ONLY,
False,
[2, 4],
4,
),
(
CUDAGraphMode.FULL_AND_PIECEWISE,
False,
True,
CUDAGraphMode.FULL,
False,
[2, 4],
4,
),
(
CUDAGraphMode.FULL_AND_PIECEWISE,
True,
True,
CUDAGraphMode.FULL_AND_PIECEWISE,
True,
[2, 4],
4,
),
],
)
def test_sequence_parallelism_requires_full_graph_compilation(
cudagraph_mode: CUDAGraphMode,
use_inductor_graph_partition: bool,
expected_enable_sp: bool,
expected_cudagraph_mode: CUDAGraphMode,
expected_piecewise_compile: bool,
expected_capture_sizes: list[int],
expected_max_size: int,
):
with patch.object(current_platform, "device_count", return_value=2):
vllm_config = VllmConfig(
parallel_config=ParallelConfig(tensor_parallel_size=2),
scheduler_config=SchedulerConfig(
max_num_seqs=128,
max_num_batched_tokens=2048,
max_model_len=2048,
is_encoder_decoder=False,
),
)
vllm_config.model_config = MagicMock(
dtype=torch.float16,
enforce_eager=False,
is_moe=False,
disable_cascade_attn=False,
get_hidden_size=MagicMock(return_value=4096),
)
vllm_config.compilation_config = CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
cudagraph_capture_sizes=[1, 2, 4, 15],
max_cudagraph_capture_size=None,
compile_sizes=["cudagraph_capture_sizes"],
use_inductor_graph_partition=use_inductor_graph_partition,
pass_config=PassConfig(
enable_sp=True,
fuse_gemm_comms=True,
fuse_norm_quant=True,
fuse_act_quant=True,
eliminate_noops=True,
sp_min_token_num=512,
),
cudagraph_mode=cudagraph_mode,
)
vllm_config.compilation_config.set_splitting_ops_for_v1(
all2all_backend=vllm_config.parallel_config.all2all_backend,
data_parallel_size=1,
)
vllm_config._set_compile_ranges()
vllm_config._set_cudagraph_sizes()
assert (
vllm_config.compilation_config.use_inductor_graph_partition
== use_inductor_graph_partition
)
assert (
bool(vllm_config.compilation_config.splitting_ops) == expected_piecewise_compile
)
assert vllm_config.compilation_config.pass_config.enable_sp == expected_enable_sp
assert (
vllm_config.compilation_config.pass_config.fuse_gemm_comms == expected_enable_sp
)
assert vllm_config.compilation_config.cudagraph_mode == expected_cudagraph_mode
assert (
vllm_config.compilation_config.cudagraph_capture_sizes == expected_capture_sizes
)
assert (
vllm_config.compilation_config.max_cudagraph_capture_size == expected_max_size
)
assert (
511 in vllm_config.compilation_config.compile_ranges_endpoints
) == expected_enable_sp
def test_cached_compilation_config(default_vllm_config):
import torch
from torch._inductor.utils import run_and_get_code