[Compile] Remove torch.compile-based sequence parallelism and AsyncTP (#60517)

This commit is contained in:
Woosuk Kwon
2026-10-08 08:28:23 -07:00
committed by GitHub
parent 072d49bb6e
commit 194bd72be7
34 changed files with 32 additions and 4797 deletions
+12 -172
View File
@@ -464,49 +464,41 @@ def test_should_split():
(
"cudagraph_capture_sizes",
"max_cudagraph_capture_size",
"tp_size",
"enable_sp",
"max_num_batched_tokens",
"cudagraph_mode",
"expected_max_size",
),
[
(None, None, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 256),
([1, 2, 4], 4, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 4),
(None, None, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 256),
([1, 2, 4], 4, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 4),
(
[1, 2, 4],
8,
1,
False,
2048,
CUDAGraphMode.FULL_AND_PIECEWISE,
ValidationError,
),
([1, 256], None, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 256),
([], None, 1, False, 2048, CUDAGraphMode.NONE, 0),
(None, 0, 1, False, 2048, CUDAGraphMode.NONE, 0),
([1, 256], None, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 256),
([], None, 2048, CUDAGraphMode.NONE, 0),
(None, 0, 2048, CUDAGraphMode.NONE, 0),
# truncated to nearest multiple of 8 or 16
(None, 257, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 256),
(None, 257, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 256),
# max_num_batched_tokens <= max_cudagraph_capture_size should always be
# captured even if not landing on a 16-stride step
(None, 2048, 1, False, 257, CUDAGraphMode.FULL_AND_PIECEWISE, 257),
(None, 2048, 257, CUDAGraphMode.FULL_AND_PIECEWISE, 257),
# max from list
([1, 2, 4, 15], None, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 15),
# SP forces full-graph compilation, sizes are filtered by TP
([1, 2, 4, 15], None, 2, True, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 4),
([1, 2, 4, 15], None, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 15),
# limited by the max_tokens
([1, 2, 4, 15], None, 1, False, 8, CUDAGraphMode.FULL_AND_PIECEWISE, 4),
([1, 2, 4, 15], None, 8, CUDAGraphMode.FULL_AND_PIECEWISE, 4),
# the list should contain at least 1 element when use cudagraph
([], None, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, ValidationError),
([], None, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, ValidationError),
# the max capturing size should be >= 1 when use cudagraph
(None, 0, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, ValidationError),
(None, 0, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, ValidationError),
],
)
def test_cudagraph_sizes_post_init(
cudagraph_capture_sizes,
max_cudagraph_capture_size,
tp_size,
enable_sp,
max_num_batched_tokens,
cudagraph_mode,
expected_max_size,
@@ -515,10 +507,7 @@ def test_cudagraph_sizes_post_init(
if expected_max_size == ValidationError:
ctx = pytest.raises(expected_max_size)
with (
ctx,
patch.object(current_platform, "device_count", return_value=tp_size),
):
with ctx:
kwargs = {}
if cudagraph_capture_sizes is not None:
kwargs["cudagraph_capture_sizes"] = cudagraph_capture_sizes
@@ -526,18 +515,15 @@ def test_cudagraph_sizes_post_init(
kwargs["max_cudagraph_capture_size"] = max_cudagraph_capture_size
compilation_config = CompilationConfig(
pass_config=PassConfig(
enable_sp=enable_sp,
fuse_norm_quant=True,
fuse_act_quant=True,
eliminate_noops=True,
sp_min_token_num=512 if enable_sp else None,
),
cudagraph_mode=cudagraph_mode,
**kwargs,
)
engine_args = EngineArgs(
model="facebook/opt-125m",
tensor_parallel_size=tp_size,
max_num_seqs=min(max_num_batched_tokens, 128),
max_num_batched_tokens=max_num_batched_tokens,
compilation_config=compilation_config,
@@ -728,35 +714,6 @@ def test_default_cudagraph_capture_sizes_keep_all_sizes_bounded():
)
def test_cudagraph_capture_sizes_respect_sequence_parallelism():
"""Sequence-parallel capture sizes stay divisible by tensor parallel size."""
compilation_config = CompilationConfig(
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
)
compilation_config.pass_config.enable_sp = True
config = _mock_config_for_cudagraph_sizes(
max_num_seqs=32,
num_speculative_tokens=16,
max_num_batched_tokens=32768,
compilation_config=compilation_config,
)
config.parallel_config = SimpleNamespace(tensor_parallel_size=2)
config.update_sizes_for_sequence_parallelism = lambda sizes: (
VllmConfig.update_sizes_for_sequence_parallelism(config, sizes)
)
with patch.object(
current_platform,
"is_device_capability_family",
return_value=False,
):
VllmConfig._set_cudagraph_sizes(config)
assert all(size % 2 == 0 for size in compilation_config.cudagraph_capture_sizes)
assert 544 not in compilation_config.cudagraph_capture_sizes
assert compilation_config.max_cudagraph_capture_size == 512
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test CUDA")
def test_real_vllm_config_caps_widest_ngram_decode_batch():
"""Real config post-init preserves the default capture ceiling."""
@@ -1046,123 +1003,6 @@ def test_blackwell_cudagraph_default(is_blackwell, expected_max_size):
)
@pytest.mark.skipif(
not current_platform.support_static_graph_mode(),
reason="Skip if not cudagraph mode supported",
)
@pytest.mark.parametrize(
(
"cudagraph_mode",
"use_inductor_graph_partition",
"expected_enable_sp",
"expected_cudagraph_mode",
"expected_piecewise_compile",
"expected_capture_sizes",
"expected_max_size",
),
[
(CUDAGraphMode.PIECEWISE, False, True, CUDAGraphMode.FULL, False, [2, 4], 4),
(
CUDAGraphMode.FULL_DECODE_ONLY,
False,
True,
CUDAGraphMode.FULL_DECODE_ONLY,
False,
[2, 4],
4,
),
(
CUDAGraphMode.FULL_AND_PIECEWISE,
False,
True,
CUDAGraphMode.FULL,
False,
[2, 4],
4,
),
(
CUDAGraphMode.FULL_AND_PIECEWISE,
True,
True,
CUDAGraphMode.FULL_AND_PIECEWISE,
True,
[2, 4],
4,
),
],
)
def test_sequence_parallelism_requires_full_graph_compilation(
cudagraph_mode: CUDAGraphMode,
use_inductor_graph_partition: bool,
expected_enable_sp: bool,
expected_cudagraph_mode: CUDAGraphMode,
expected_piecewise_compile: bool,
expected_capture_sizes: list[int],
expected_max_size: int,
):
with patch.object(current_platform, "device_count", return_value=2):
vllm_config = VllmConfig(
parallel_config=ParallelConfig(tensor_parallel_size=2),
scheduler_config=SchedulerConfig(
max_num_seqs=128,
max_num_batched_tokens=2048,
max_model_len=2048,
is_encoder_decoder=False,
),
)
vllm_config.model_config = MagicMock(
dtype=torch.float16,
enforce_eager=False,
is_moe=False,
disable_cascade_attn=False,
get_hidden_size=MagicMock(return_value=4096),
)
vllm_config.compilation_config = CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
cudagraph_capture_sizes=[1, 2, 4, 15],
max_cudagraph_capture_size=None,
compile_sizes=["cudagraph_capture_sizes"],
use_inductor_graph_partition=use_inductor_graph_partition,
pass_config=PassConfig(
enable_sp=True,
fuse_gemm_comms=True,
fuse_norm_quant=True,
fuse_act_quant=True,
eliminate_noops=True,
sp_min_token_num=512,
),
cudagraph_mode=cudagraph_mode,
)
vllm_config.compilation_config.set_splitting_ops_for_v1(
all2all_backend=vllm_config.parallel_config.all2all_backend,
data_parallel_size=1,
)
vllm_config._set_compile_ranges()
vllm_config._set_cudagraph_sizes()
assert (
vllm_config.compilation_config.use_inductor_graph_partition
== use_inductor_graph_partition
)
assert (
bool(vllm_config.compilation_config.splitting_ops) == expected_piecewise_compile
)
assert vllm_config.compilation_config.pass_config.enable_sp == expected_enable_sp
assert (
vllm_config.compilation_config.pass_config.fuse_gemm_comms == expected_enable_sp
)
assert vllm_config.compilation_config.cudagraph_mode == expected_cudagraph_mode
assert (
vllm_config.compilation_config.cudagraph_capture_sizes == expected_capture_sizes
)
assert (
vllm_config.compilation_config.max_cudagraph_capture_size == expected_max_size
)
assert (
511 in vllm_config.compilation_config.compile_ranges_endpoints
) == expected_enable_sp
def test_cached_compilation_config(default_vllm_config):
import torch
from torch._inductor.utils import run_and_get_code