[Compile] Remove torch.compile-based sequence parallelism and AsyncTP (#60517)

This commit is contained in:
Woosuk Kwon
2026-10-08 08:28:23 -07:00
committed by GitHub
parent 072d49bb6e
commit 194bd72be7
34 changed files with 32 additions and 4797 deletions
+3 -57
View File
@@ -133,12 +133,6 @@ class PassConfig:
"""Fuse the custom Attention and MLAAttention + quant ops."""
eliminate_noops: bool = Field(default=True)
"""Eliminate no-op ops."""
enable_sp: bool = None # type: ignore[assignment]
"""Enable sequence parallelism. Requires TP>1. Automatically disabled
if the model's hidden_size is too small for SP to be beneficial
(threshold is device-capability dependent)."""
fuse_gemm_comms: bool = None # type: ignore[assignment]
"""Enable async TP."""
fuse_allreduce_rms: bool = None # type: ignore[assignment]
"""Enable flashinfer allreduce fusion."""
enable_qk_norm_rope_fusion: bool = None # type: ignore[assignment]
@@ -183,12 +177,6 @@ class PassConfig:
8: 1, # 1MB
},
}, where key is the device capability"""
sp_min_token_num: int | None = None
"""The minimum number of tokens above which vllm should use
sequence parallelism. Specified as an integer token count.
Unspecified will fallback to default values which are compute
capability and world size dependent."""
# TODO(luka) better pass enabling system.
def flashinfer_max_size(self, world_size: int) -> int | None:
@@ -231,8 +219,6 @@ class PassConfig:
"fuse_norm_quant",
"fuse_act_quant",
"fuse_attn_quant",
"enable_sp",
"fuse_gemm_comms",
"fuse_allreduce_rms",
"fuse_act_padding",
"fuse_mla_dual_rms_norm",
@@ -1244,25 +1230,6 @@ class CompilationConfig:
self.cudagraph_mode = CUDAGraphMode.FULL
self.splitting_ops = []
if (
not self.use_inductor_graph_partition
and (self.pass_config.enable_sp or self.pass_config.fuse_gemm_comms)
and self.splitting_ops
):
logger.warning_once(
"Sequence parallelism requires full-graph compilation when "
"use_inductor_graph_partition is off. Setting splitting_ops "
"to an empty list to preserve SP and async TP."
)
self.splitting_ops = []
if self.cudagraph_mode.has_piecewise_cudagraphs():
logger.warning_once(
"Sequence parallelism is incompatible with piecewise "
"cudagraph when use_inductor_graph_partition is off. "
"Setting cudagraph_mode to FULL."
)
self.cudagraph_mode = CUDAGraphMode.FULL
# Disable CUDA graphs for DeepEP high-throughput since its not CG compatible
if (
all2all_backend == "deepep_high_throughput"
@@ -1403,7 +1370,6 @@ class CompilationConfig:
min_cg_attn_backend: str | None,
uniform_decode_query_len: int = 1,
use_v2_model_runner: bool = False,
tensor_parallel_size: int = 1,
kv_cache_config: "KVCacheConfig | None" = None,
max_num_reqs: int | None = None,
is_profiling: bool = False,
@@ -1530,10 +1496,7 @@ class CompilationConfig:
and cudagraph_mode.decode_mode() == CUDAGraphMode.FULL
and uniform_decode_query_len > 1
):
self.adjust_cudagraph_sizes_for_spec_decode(
uniform_decode_query_len,
tensor_parallel_size,
)
self.adjust_cudagraph_sizes_for_spec_decode(uniform_decode_query_len)
# For Mamba models with FULL decode cudagraphs, each decode
# sequence needs one Mamba cache block. The decode cudagraph
@@ -1562,25 +1525,8 @@ class CompilationConfig:
self.cudagraph_mode = cudagraph_mode
return cudagraph_mode
def adjust_cudagraph_sizes_for_spec_decode(
self, uniform_decode_query_len: int, tensor_parallel_size: int
):
def adjust_cudagraph_sizes_for_spec_decode(self, uniform_decode_query_len: int):
multiple_of = uniform_decode_query_len
if tensor_parallel_size > 1 and self.pass_config.enable_sp:
multiple_of = max(uniform_decode_query_len, tensor_parallel_size)
if (
multiple_of % uniform_decode_query_len != 0
or multiple_of % tensor_parallel_size != 0
):
raise ValueError(
f"Can't determine cudagraph shapes that are both a "
f"multiple of {uniform_decode_query_len} "
f"(num_speculative_tokens + 1) required by spec-decode "
f"and {tensor_parallel_size} (tensor_parallel_size) "
f"required by sequence parallelism please adjust "
f"num_speculative_tokens or disable sequence parallelism"
)
if not self.cudagraph_capture_sizes or multiple_of <= 1:
return
@@ -1600,7 +1546,7 @@ class CompilationConfig:
if len(rounded_sizes) == 0:
raise ValueError(
f"No valid cudagraph sizes after rounding to multiple of {multiple_of} "
f"(num_speculative_tokens + 1 or tp if sequence parallelism is enabled)"
f"(num_speculative_tokens + 1)"
f" please adjust num_speculative_tokens ({uniform_decode_query_len - 1}"
f") or max_cudagraph_capture_size ({self.max_cudagraph_capture_size})"
f" or cudagraph_capture_sizes ({self.cudagraph_capture_sizes})"
-159
View File
@@ -145,12 +145,10 @@ class OptimizationLevel(IntEnum):
PerformanceMode = Literal["balanced", "interactivity", "throughput"]
IS_QUANTIZED = False
IS_DENSE = False
# The optimizations that depend on these properties currently set to False
# in all cases.
# if model_config is not None:
# IS_QUANTIZED = lambda c: c.model_config.is_quantized()
# IS_DENSE = lambda c: not c.model_config.is_model_moe()
# See https://github.com/vllm-project/vllm/issues/25689.
@@ -259,8 +257,6 @@ OPTIMIZATION_LEVEL_00 = {
"fuse_act_quant": False,
"fuse_allreduce_rms": False,
"fuse_attn_quant": False,
"enable_sp": False,
"fuse_gemm_comms": False,
"fuse_act_padding": False,
"fuse_mla_dual_rms_norm": False,
"fuse_rope_kvcache": False,
@@ -282,8 +278,6 @@ OPTIMIZATION_LEVEL_01 = {
"fuse_act_quant": enable_act_fusion,
"fuse_allreduce_rms": False,
"fuse_attn_quant": False,
"enable_sp": False,
"fuse_gemm_comms": False,
"fuse_act_padding": enable_norm_pad_fusion,
"fuse_mla_dual_rms_norm": enable_mla_dual_rms_norm_fusion,
"fuse_rope_kvcache": False,
@@ -305,8 +299,6 @@ OPTIMIZATION_LEVEL_02 = {
"fuse_act_quant": enable_act_fusion,
"fuse_allreduce_rms": enable_allreduce_rms_fusion,
"fuse_attn_quant": IS_QUANTIZED,
"enable_sp": IS_DENSE,
"fuse_gemm_comms": IS_DENSE,
"fuse_act_padding": enable_norm_pad_fusion,
"fuse_mla_dual_rms_norm": enable_mla_dual_rms_norm_fusion,
"fuse_rope_kvcache": enable_rope_kvcache_fusion,
@@ -328,8 +320,6 @@ OPTIMIZATION_LEVEL_03 = {
"fuse_act_quant": enable_act_fusion,
"fuse_allreduce_rms": enable_allreduce_rms_fusion,
"fuse_attn_quant": IS_QUANTIZED,
"enable_sp": IS_DENSE,
"fuse_gemm_comms": IS_DENSE,
"fuse_act_padding": enable_norm_pad_fusion,
"fuse_mla_dual_rms_norm": enable_mla_dual_rms_norm_fusion,
"fuse_rope_kvcache": enable_rope_kvcache_fusion,
@@ -828,9 +818,6 @@ class VllmConfig:
or model.dtype != torch.bfloat16
or model.quantization is not None
or not current_platform.is_cuda()
# Sequence parallelism / async TP are torch.compile passes.
or self.compilation_config.pass_config.enable_sp
or self.compilation_config.pass_config.fuse_gemm_comms
):
return False
family = bi._get_tuned_matmul_arch_family(
@@ -1770,26 +1757,6 @@ class VllmConfig:
)
self.compilation_config.mode = CompilationMode.NONE
# TODO: This is a stopgap that turns sequence parallelism / async TP
# off under batch invariance. Make the sequence-parallel reduce-scatter
# path batch-invariant instead so they can stay enabled (#56370).
# Sequence parallelism / async TP rewrite all_reduce + rms_norm into a
# reduce-scatter whose reduction order depends on the batch, so they
# are not batch-invariant. Decide this before the breakable-CUDA-graph
# default below, which declines to auto-enable when they are on.
pass_config = self.compilation_config.pass_config
if envs.VLLM_BATCH_INVARIANT and (
pass_config.enable_sp or pass_config.fuse_gemm_comms
):
logger.warning_once(
"Disabling sequence parallelism and async TP "
"(pass_config.enable_sp / fuse_gemm_comms) when "
"VLLM_BATCH_INVARIANT is enabled: the reduce-scatter path "
"is not batch-invariant (see vllm-project/vllm#56370)."
)
pass_config.enable_sp = False
pass_config.fuse_gemm_comms = False
breakable_cudagraph_enabled = self._maybe_enable_breakable_cudagraph()
if not breakable_cudagraph_enabled and (
@@ -1920,38 +1887,6 @@ class VllmConfig:
breakable_cudagraph_enabled=breakable_cudagraph_enabled
)
# async tp is built on top of sequence parallelism and requires it.
pass_config = self.compilation_config.pass_config
if pass_config.fuse_gemm_comms:
pass_config.enable_sp = True
if pass_config.enable_sp:
if self.parallel_config.tensor_parallel_size == 1:
logger.warning_once("Sequence Parallelism requires TP>1, disabling")
pass_config.enable_sp = False
pass_config.fuse_gemm_comms = False
else:
if pass_config.sp_min_token_num is None:
from vllm.compilation.passes.fusion.sequence_parallelism import (
get_sequence_parallelism_threshold,
)
tp_size = self.parallel_config.tensor_parallel_size
hidden_size = self.model_config.get_hidden_size()
assert isinstance(self.model_config.dtype, torch.dtype)
element_size = self.model_config.dtype.itemsize
pass_config.sp_min_token_num = get_sequence_parallelism_threshold(
hidden_size, tp_size, element_size
)
if pass_config.sp_min_token_num is None:
logger.warning_once(
"Model hidden_size too small for the SP "
"threshold heuristic, disabling. To force SP, "
"set pass_config.sp_min_token_num manually."
)
pass_config.enable_sp = False
pass_config.fuse_gemm_comms = False
from vllm.utils.torch_utils import HAS_OPAQUE_TYPE
if HAS_OPAQUE_TYPE:
@@ -2184,30 +2119,6 @@ class VllmConfig:
data_parallel_size=effective_dp_size,
)
if self.compilation_config.pass_config.enable_sp:
# With pipeline parallelism, native rms norm tracing errors due to
# incorrect residual shape.
# Use custom rms norm to unblock. In the future,
# the pass will operate on higher-level IR to avoid the issue.
# TODO: https://github.com/vllm-project/vllm/issues/27894
if self.compilation_config.mode != CompilationMode.VLLM_COMPILE:
logger.warning_once(
"Sequence parallelism is enabled, but running in wrong "
"vllm compile mode: %s.",
self.compilation_config.mode,
)
if self.parallel_config.pipeline_parallel_size > 1:
if "-rms_norm" not in self.compilation_config.custom_ops:
self.compilation_config.custom_ops.append("+rms_norm")
else:
logger.warning_once(
"Sequence parallelism not supported with "
"native rms_norm when using %s, "
"this will likely lead to an error.",
"pipeline parallelism",
)
# final check of cudagraph mode after all possible updates
if current_platform.is_cuda_alike():
if (
@@ -2395,29 +2306,6 @@ class VllmConfig:
# Log the custom passes that are enabled
self.compilation_config.pass_config.log_enabled_passes()
def update_sizes_for_sequence_parallelism(self, possible_sizes: list) -> list:
# remove the sizes that not multiple of tp_size when
# enable sequence parallelism
removed_sizes = [
size
for size in possible_sizes
if size % self.parallel_config.tensor_parallel_size != 0
]
if removed_sizes:
logger.warning(
"Batch sizes %s are removed because they are not "
"multiple of tp_size %d when "
"sequence parallelism is enabled",
removed_sizes,
self.parallel_config.tensor_parallel_size,
)
return [
size
for size in possible_sizes
if size % self.parallel_config.tensor_parallel_size == 0
]
def _set_max_num_scheduled_tokens(self):
"""In most cases, the scheduler may schedule a batch with as many tokens as the
worker is configured to handle.
@@ -2665,16 +2553,6 @@ class VllmConfig:
# de-duplicate and sort the sizes
cudagraph_capture_sizes = sorted(set(cudagraph_capture_sizes))
if (
self.parallel_config.tensor_parallel_size > 1
and self.compilation_config.pass_config.enable_sp
):
# Sequence parallelism only captures TP-divisible sizes, so a
# wider non-divisible decode batch cannot be captured under SP.
cudagraph_capture_sizes = self.update_sizes_for_sequence_parallelism(
cudagraph_capture_sizes
)
# user-specific compilation_config.max_cudagraph_capture_size get
# truncated to valid_max_size when they are inconsistent.
valid_max_size = (
@@ -2764,37 +2642,6 @@ class VllmConfig:
"allreduce-rms fusion will be enabled for all num_tokens."
)
# Add the compile ranges for sequence parallelism
if compilation_config.pass_config.enable_sp:
pass_config = compilation_config.pass_config
# Calculate min_token_num if not explicitly provided
# User override works regardless of hidden_size
if pass_config.sp_min_token_num is None:
from vllm.compilation.passes.fusion.sequence_parallelism import (
get_sequence_parallelism_threshold,
)
tp_size = self.parallel_config.tensor_parallel_size
hidden_size = self.model_config.get_hidden_size()
assert isinstance(self.model_config.dtype, torch.dtype)
element_size = self.model_config.dtype.itemsize
pass_config.sp_min_token_num = get_sequence_parallelism_threshold(
hidden_size, tp_size, element_size
)
min_token_num = pass_config.sp_min_token_num
max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens
if min_token_num is not None and (
max_num_batched_tokens is not None
and min_token_num < max_num_batched_tokens
and min_token_num > 1
):
# Add endpoint at min_token_num - 1 to ensure SP applies
# starting from min_token_num
# This creates ranges: [1, min-1] (no SP), [min, max] (SP applies)
computed_compile_ranges_endpoints.append(min_token_num - 1)
if compilation_config.pass_config.fuse_rope_kvcache:
max_token_num = (
compilation_config.pass_config.rope_kvcache_fusion_max_token_num
@@ -3124,12 +2971,6 @@ class VllmConfig:
unsupported: list[str] = []
speculative_config = self.speculative_config
if (
self.compilation_config.pass_config.enable_sp
and self.parallel_config.tensor_parallel_size > 1
):
unsupported.append("sequence parallelism")
# V2 does not implement the external_launcher (torchrun) PP-output
# broadcast that V1 uses to keep all ranks in sync (broadcast_pp_output).
if (