[torch.compile]: Disable Sequence Parallelism (SP) for piecewise compilation (#38373)
Signed-off-by: SouthWest7 <am1ao@qq.com> Signed-off-by: Xinan Miao <1403572259@qq.com> Co-authored-by: SouthWest7 <am1ao@qq.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Wang Xingran <72983099+wangxingran222@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
SouthWest7
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
OpenAI Codex
Wang Xingran
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
b39c266dae
commit
32e45636e3
@@ -261,6 +261,8 @@ def _compare_sp(
|
||||
},
|
||||
"use_inductor_graph_partition": use_inductor_graph_partition,
|
||||
}
|
||||
if not use_inductor_graph_partition:
|
||||
compilation_config["splitting_ops"] = []
|
||||
|
||||
tp_sp_args = [
|
||||
*common_args,
|
||||
|
||||
@@ -19,6 +19,7 @@ from vllm.config import (
|
||||
VllmConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.config.utils import Range
|
||||
from vllm.distributed import (
|
||||
tensor_model_parallel_all_gather,
|
||||
tensor_model_parallel_reduce_scatter,
|
||||
@@ -288,6 +289,22 @@ def test_async_tp_pass_replace(
|
||||
run_torch_spawn(async_tp_pass_on_test_model, num_processes)
|
||||
|
||||
|
||||
def test_async_tp_pass_requires_full_graph_compilation():
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.compilation_config.use_inductor_graph_partition = False
|
||||
vllm_config.compilation_config.splitting_ops = [
|
||||
"vllm::unified_attention_with_output"
|
||||
]
|
||||
|
||||
async_tp_pass = object.__new__(AsyncTPPass)
|
||||
async_tp_pass.compilation_config = vllm_config.compilation_config
|
||||
|
||||
with pytest.raises(
|
||||
AssertionError, match="AsyncTPPass requires full-graph compilation"
|
||||
):
|
||||
async_tp_pass.is_applicable_for_range(Range(start=8, end=8))
|
||||
|
||||
|
||||
def async_tp_pass_on_test_model(
|
||||
local_rank: int,
|
||||
world_size: int,
|
||||
|
||||
@@ -22,6 +22,7 @@ from vllm.config import (
|
||||
get_current_vllm_config,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.config.utils import Range
|
||||
from vllm.distributed import tensor_model_parallel_all_reduce
|
||||
from vllm.distributed.parallel_state import (
|
||||
init_distributed_environment,
|
||||
@@ -216,6 +217,24 @@ def test_sequence_parallelism_pass(
|
||||
run_torch_spawn(sequence_parallelism_pass_on_test_model, num_processes)
|
||||
|
||||
|
||||
def test_sequence_parallelism_pass_requires_full_graph_compilation():
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.compilation_config.use_inductor_graph_partition = False
|
||||
vllm_config.compilation_config.splitting_ops = [
|
||||
"vllm::unified_attention_with_output"
|
||||
]
|
||||
|
||||
sequence_parallelism_pass = object.__new__(SequenceParallelismPass)
|
||||
sequence_parallelism_pass.compilation_config = vllm_config.compilation_config
|
||||
sequence_parallelism_pass.min_token_num = 1
|
||||
|
||||
with pytest.raises(
|
||||
AssertionError,
|
||||
match="SequenceParallelismPass requires full-graph compilation",
|
||||
):
|
||||
sequence_parallelism_pass.is_applicable_for_range(Range(start=8, end=8))
|
||||
|
||||
|
||||
def sequence_parallelism_pass_on_test_model(
|
||||
local_rank: int,
|
||||
world_size: int,
|
||||
|
||||
@@ -407,7 +407,7 @@ def test_should_split():
|
||||
(None, 257, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 256),
|
||||
# max from list
|
||||
([1, 2, 4, 15], None, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 15),
|
||||
# filtered out 15 due to SP
|
||||
# SP forces full-graph compilation, sizes are filtered by TP
|
||||
([1, 2, 4, 15], None, 2, True, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 4),
|
||||
# limited by the max_tokens
|
||||
([1, 2, 4, 15], None, 1, False, 8, CUDAGraphMode.FULL_AND_PIECEWISE, 4),
|
||||
@@ -465,6 +465,123 @@ def test_cudagraph_sizes_post_init(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.support_static_graph_mode(),
|
||||
reason="Skip if not cudagraph mode supported",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"cudagraph_mode",
|
||||
"use_inductor_graph_partition",
|
||||
"expected_enable_sp",
|
||||
"expected_cudagraph_mode",
|
||||
"expected_piecewise_compile",
|
||||
"expected_capture_sizes",
|
||||
"expected_max_size",
|
||||
),
|
||||
[
|
||||
(CUDAGraphMode.PIECEWISE, False, True, CUDAGraphMode.FULL, False, [2, 4], 4),
|
||||
(
|
||||
CUDAGraphMode.FULL_DECODE_ONLY,
|
||||
False,
|
||||
True,
|
||||
CUDAGraphMode.FULL_DECODE_ONLY,
|
||||
False,
|
||||
[2, 4],
|
||||
4,
|
||||
),
|
||||
(
|
||||
CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
False,
|
||||
True,
|
||||
CUDAGraphMode.FULL,
|
||||
False,
|
||||
[2, 4],
|
||||
4,
|
||||
),
|
||||
(
|
||||
CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
True,
|
||||
True,
|
||||
CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
True,
|
||||
[2, 4],
|
||||
4,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_sequence_parallelism_requires_full_graph_compilation(
|
||||
cudagraph_mode: CUDAGraphMode,
|
||||
use_inductor_graph_partition: bool,
|
||||
expected_enable_sp: bool,
|
||||
expected_cudagraph_mode: CUDAGraphMode,
|
||||
expected_piecewise_compile: bool,
|
||||
expected_capture_sizes: list[int],
|
||||
expected_max_size: int,
|
||||
):
|
||||
with patch.object(current_platform, "device_count", return_value=2):
|
||||
vllm_config = VllmConfig(
|
||||
parallel_config=ParallelConfig(tensor_parallel_size=2),
|
||||
scheduler_config=SchedulerConfig(
|
||||
max_num_seqs=128,
|
||||
max_num_batched_tokens=2048,
|
||||
max_model_len=2048,
|
||||
is_encoder_decoder=False,
|
||||
),
|
||||
)
|
||||
vllm_config.model_config = MagicMock(
|
||||
dtype=torch.float16,
|
||||
enforce_eager=False,
|
||||
is_moe=False,
|
||||
disable_cascade_attn=False,
|
||||
get_hidden_size=MagicMock(return_value=4096),
|
||||
)
|
||||
vllm_config.compilation_config = CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
cudagraph_capture_sizes=[1, 2, 4, 15],
|
||||
max_cudagraph_capture_size=None,
|
||||
compile_sizes=["cudagraph_capture_sizes"],
|
||||
use_inductor_graph_partition=use_inductor_graph_partition,
|
||||
pass_config=PassConfig(
|
||||
enable_sp=True,
|
||||
fuse_gemm_comms=True,
|
||||
fuse_norm_quant=True,
|
||||
fuse_act_quant=True,
|
||||
eliminate_noops=True,
|
||||
sp_min_token_num=512,
|
||||
),
|
||||
cudagraph_mode=cudagraph_mode,
|
||||
)
|
||||
vllm_config.compilation_config.set_splitting_ops_for_v1(
|
||||
all2all_backend=vllm_config.parallel_config.all2all_backend,
|
||||
data_parallel_size=1,
|
||||
)
|
||||
vllm_config._set_compile_ranges()
|
||||
vllm_config._set_cudagraph_sizes()
|
||||
|
||||
assert (
|
||||
vllm_config.compilation_config.use_inductor_graph_partition
|
||||
== use_inductor_graph_partition
|
||||
)
|
||||
assert (
|
||||
bool(vllm_config.compilation_config.splitting_ops) == expected_piecewise_compile
|
||||
)
|
||||
assert vllm_config.compilation_config.pass_config.enable_sp == expected_enable_sp
|
||||
assert (
|
||||
vllm_config.compilation_config.pass_config.fuse_gemm_comms == expected_enable_sp
|
||||
)
|
||||
assert vllm_config.compilation_config.cudagraph_mode == expected_cudagraph_mode
|
||||
assert (
|
||||
vllm_config.compilation_config.cudagraph_capture_sizes == expected_capture_sizes
|
||||
)
|
||||
assert (
|
||||
vllm_config.compilation_config.max_cudagraph_capture_size == expected_max_size
|
||||
)
|
||||
assert (
|
||||
511 in vllm_config.compilation_config.compile_ranges_endpoints
|
||||
) == expected_enable_sp
|
||||
|
||||
|
||||
def test_cached_compilation_config(default_vllm_config):
|
||||
import torch
|
||||
from torch._inductor.utils import run_and_get_code
|
||||
|
||||
Reference in New Issue
Block a user