[Compile] Remove torch.compile-based sequence parallelism and AsyncTP (#60517)
This commit is contained in:
@@ -133,12 +133,6 @@ class PassConfig:
|
||||
"""Fuse the custom Attention and MLAAttention + quant ops."""
|
||||
eliminate_noops: bool = Field(default=True)
|
||||
"""Eliminate no-op ops."""
|
||||
enable_sp: bool = None # type: ignore[assignment]
|
||||
"""Enable sequence parallelism. Requires TP>1. Automatically disabled
|
||||
if the model's hidden_size is too small for SP to be beneficial
|
||||
(threshold is device-capability dependent)."""
|
||||
fuse_gemm_comms: bool = None # type: ignore[assignment]
|
||||
"""Enable async TP."""
|
||||
fuse_allreduce_rms: bool = None # type: ignore[assignment]
|
||||
"""Enable flashinfer allreduce fusion."""
|
||||
enable_qk_norm_rope_fusion: bool = None # type: ignore[assignment]
|
||||
@@ -183,12 +177,6 @@ class PassConfig:
|
||||
8: 1, # 1MB
|
||||
},
|
||||
}, where key is the device capability"""
|
||||
sp_min_token_num: int | None = None
|
||||
"""The minimum number of tokens above which vllm should use
|
||||
sequence parallelism. Specified as an integer token count.
|
||||
Unspecified will fallback to default values which are compute
|
||||
capability and world size dependent."""
|
||||
|
||||
# TODO(luka) better pass enabling system.
|
||||
|
||||
def flashinfer_max_size(self, world_size: int) -> int | None:
|
||||
@@ -231,8 +219,6 @@ class PassConfig:
|
||||
"fuse_norm_quant",
|
||||
"fuse_act_quant",
|
||||
"fuse_attn_quant",
|
||||
"enable_sp",
|
||||
"fuse_gemm_comms",
|
||||
"fuse_allreduce_rms",
|
||||
"fuse_act_padding",
|
||||
"fuse_mla_dual_rms_norm",
|
||||
@@ -1244,25 +1230,6 @@ class CompilationConfig:
|
||||
self.cudagraph_mode = CUDAGraphMode.FULL
|
||||
self.splitting_ops = []
|
||||
|
||||
if (
|
||||
not self.use_inductor_graph_partition
|
||||
and (self.pass_config.enable_sp or self.pass_config.fuse_gemm_comms)
|
||||
and self.splitting_ops
|
||||
):
|
||||
logger.warning_once(
|
||||
"Sequence parallelism requires full-graph compilation when "
|
||||
"use_inductor_graph_partition is off. Setting splitting_ops "
|
||||
"to an empty list to preserve SP and async TP."
|
||||
)
|
||||
self.splitting_ops = []
|
||||
if self.cudagraph_mode.has_piecewise_cudagraphs():
|
||||
logger.warning_once(
|
||||
"Sequence parallelism is incompatible with piecewise "
|
||||
"cudagraph when use_inductor_graph_partition is off. "
|
||||
"Setting cudagraph_mode to FULL."
|
||||
)
|
||||
self.cudagraph_mode = CUDAGraphMode.FULL
|
||||
|
||||
# Disable CUDA graphs for DeepEP high-throughput since its not CG compatible
|
||||
if (
|
||||
all2all_backend == "deepep_high_throughput"
|
||||
@@ -1403,7 +1370,6 @@ class CompilationConfig:
|
||||
min_cg_attn_backend: str | None,
|
||||
uniform_decode_query_len: int = 1,
|
||||
use_v2_model_runner: bool = False,
|
||||
tensor_parallel_size: int = 1,
|
||||
kv_cache_config: "KVCacheConfig | None" = None,
|
||||
max_num_reqs: int | None = None,
|
||||
is_profiling: bool = False,
|
||||
@@ -1530,10 +1496,7 @@ class CompilationConfig:
|
||||
and cudagraph_mode.decode_mode() == CUDAGraphMode.FULL
|
||||
and uniform_decode_query_len > 1
|
||||
):
|
||||
self.adjust_cudagraph_sizes_for_spec_decode(
|
||||
uniform_decode_query_len,
|
||||
tensor_parallel_size,
|
||||
)
|
||||
self.adjust_cudagraph_sizes_for_spec_decode(uniform_decode_query_len)
|
||||
|
||||
# For Mamba models with FULL decode cudagraphs, each decode
|
||||
# sequence needs one Mamba cache block. The decode cudagraph
|
||||
@@ -1562,25 +1525,8 @@ class CompilationConfig:
|
||||
self.cudagraph_mode = cudagraph_mode
|
||||
return cudagraph_mode
|
||||
|
||||
def adjust_cudagraph_sizes_for_spec_decode(
|
||||
self, uniform_decode_query_len: int, tensor_parallel_size: int
|
||||
):
|
||||
def adjust_cudagraph_sizes_for_spec_decode(self, uniform_decode_query_len: int):
|
||||
multiple_of = uniform_decode_query_len
|
||||
if tensor_parallel_size > 1 and self.pass_config.enable_sp:
|
||||
multiple_of = max(uniform_decode_query_len, tensor_parallel_size)
|
||||
if (
|
||||
multiple_of % uniform_decode_query_len != 0
|
||||
or multiple_of % tensor_parallel_size != 0
|
||||
):
|
||||
raise ValueError(
|
||||
f"Can't determine cudagraph shapes that are both a "
|
||||
f"multiple of {uniform_decode_query_len} "
|
||||
f"(num_speculative_tokens + 1) required by spec-decode "
|
||||
f"and {tensor_parallel_size} (tensor_parallel_size) "
|
||||
f"required by sequence parallelism please adjust "
|
||||
f"num_speculative_tokens or disable sequence parallelism"
|
||||
)
|
||||
|
||||
if not self.cudagraph_capture_sizes or multiple_of <= 1:
|
||||
return
|
||||
|
||||
@@ -1600,7 +1546,7 @@ class CompilationConfig:
|
||||
if len(rounded_sizes) == 0:
|
||||
raise ValueError(
|
||||
f"No valid cudagraph sizes after rounding to multiple of {multiple_of} "
|
||||
f"(num_speculative_tokens + 1 or tp if sequence parallelism is enabled)"
|
||||
f"(num_speculative_tokens + 1)"
|
||||
f" please adjust num_speculative_tokens ({uniform_decode_query_len - 1}"
|
||||
f") or max_cudagraph_capture_size ({self.max_cudagraph_capture_size})"
|
||||
f" or cudagraph_capture_sizes ({self.cudagraph_capture_sizes})"
|
||||
|
||||
@@ -145,12 +145,10 @@ class OptimizationLevel(IntEnum):
|
||||
PerformanceMode = Literal["balanced", "interactivity", "throughput"]
|
||||
|
||||
IS_QUANTIZED = False
|
||||
IS_DENSE = False
|
||||
# The optimizations that depend on these properties currently set to False
|
||||
# in all cases.
|
||||
# if model_config is not None:
|
||||
# IS_QUANTIZED = lambda c: c.model_config.is_quantized()
|
||||
# IS_DENSE = lambda c: not c.model_config.is_model_moe()
|
||||
# See https://github.com/vllm-project/vllm/issues/25689.
|
||||
|
||||
|
||||
@@ -259,8 +257,6 @@ OPTIMIZATION_LEVEL_00 = {
|
||||
"fuse_act_quant": False,
|
||||
"fuse_allreduce_rms": False,
|
||||
"fuse_attn_quant": False,
|
||||
"enable_sp": False,
|
||||
"fuse_gemm_comms": False,
|
||||
"fuse_act_padding": False,
|
||||
"fuse_mla_dual_rms_norm": False,
|
||||
"fuse_rope_kvcache": False,
|
||||
@@ -282,8 +278,6 @@ OPTIMIZATION_LEVEL_01 = {
|
||||
"fuse_act_quant": enable_act_fusion,
|
||||
"fuse_allreduce_rms": False,
|
||||
"fuse_attn_quant": False,
|
||||
"enable_sp": False,
|
||||
"fuse_gemm_comms": False,
|
||||
"fuse_act_padding": enable_norm_pad_fusion,
|
||||
"fuse_mla_dual_rms_norm": enable_mla_dual_rms_norm_fusion,
|
||||
"fuse_rope_kvcache": False,
|
||||
@@ -305,8 +299,6 @@ OPTIMIZATION_LEVEL_02 = {
|
||||
"fuse_act_quant": enable_act_fusion,
|
||||
"fuse_allreduce_rms": enable_allreduce_rms_fusion,
|
||||
"fuse_attn_quant": IS_QUANTIZED,
|
||||
"enable_sp": IS_DENSE,
|
||||
"fuse_gemm_comms": IS_DENSE,
|
||||
"fuse_act_padding": enable_norm_pad_fusion,
|
||||
"fuse_mla_dual_rms_norm": enable_mla_dual_rms_norm_fusion,
|
||||
"fuse_rope_kvcache": enable_rope_kvcache_fusion,
|
||||
@@ -328,8 +320,6 @@ OPTIMIZATION_LEVEL_03 = {
|
||||
"fuse_act_quant": enable_act_fusion,
|
||||
"fuse_allreduce_rms": enable_allreduce_rms_fusion,
|
||||
"fuse_attn_quant": IS_QUANTIZED,
|
||||
"enable_sp": IS_DENSE,
|
||||
"fuse_gemm_comms": IS_DENSE,
|
||||
"fuse_act_padding": enable_norm_pad_fusion,
|
||||
"fuse_mla_dual_rms_norm": enable_mla_dual_rms_norm_fusion,
|
||||
"fuse_rope_kvcache": enable_rope_kvcache_fusion,
|
||||
@@ -828,9 +818,6 @@ class VllmConfig:
|
||||
or model.dtype != torch.bfloat16
|
||||
or model.quantization is not None
|
||||
or not current_platform.is_cuda()
|
||||
# Sequence parallelism / async TP are torch.compile passes.
|
||||
or self.compilation_config.pass_config.enable_sp
|
||||
or self.compilation_config.pass_config.fuse_gemm_comms
|
||||
):
|
||||
return False
|
||||
family = bi._get_tuned_matmul_arch_family(
|
||||
@@ -1770,26 +1757,6 @@ class VllmConfig:
|
||||
)
|
||||
self.compilation_config.mode = CompilationMode.NONE
|
||||
|
||||
# TODO: This is a stopgap that turns sequence parallelism / async TP
|
||||
# off under batch invariance. Make the sequence-parallel reduce-scatter
|
||||
# path batch-invariant instead so they can stay enabled (#56370).
|
||||
# Sequence parallelism / async TP rewrite all_reduce + rms_norm into a
|
||||
# reduce-scatter whose reduction order depends on the batch, so they
|
||||
# are not batch-invariant. Decide this before the breakable-CUDA-graph
|
||||
# default below, which declines to auto-enable when they are on.
|
||||
pass_config = self.compilation_config.pass_config
|
||||
if envs.VLLM_BATCH_INVARIANT and (
|
||||
pass_config.enable_sp or pass_config.fuse_gemm_comms
|
||||
):
|
||||
logger.warning_once(
|
||||
"Disabling sequence parallelism and async TP "
|
||||
"(pass_config.enable_sp / fuse_gemm_comms) when "
|
||||
"VLLM_BATCH_INVARIANT is enabled: the reduce-scatter path "
|
||||
"is not batch-invariant (see vllm-project/vllm#56370)."
|
||||
)
|
||||
pass_config.enable_sp = False
|
||||
pass_config.fuse_gemm_comms = False
|
||||
|
||||
breakable_cudagraph_enabled = self._maybe_enable_breakable_cudagraph()
|
||||
|
||||
if not breakable_cudagraph_enabled and (
|
||||
@@ -1920,38 +1887,6 @@ class VllmConfig:
|
||||
breakable_cudagraph_enabled=breakable_cudagraph_enabled
|
||||
)
|
||||
|
||||
# async tp is built on top of sequence parallelism and requires it.
|
||||
pass_config = self.compilation_config.pass_config
|
||||
if pass_config.fuse_gemm_comms:
|
||||
pass_config.enable_sp = True
|
||||
if pass_config.enable_sp:
|
||||
if self.parallel_config.tensor_parallel_size == 1:
|
||||
logger.warning_once("Sequence Parallelism requires TP>1, disabling")
|
||||
pass_config.enable_sp = False
|
||||
pass_config.fuse_gemm_comms = False
|
||||
else:
|
||||
if pass_config.sp_min_token_num is None:
|
||||
from vllm.compilation.passes.fusion.sequence_parallelism import (
|
||||
get_sequence_parallelism_threshold,
|
||||
)
|
||||
|
||||
tp_size = self.parallel_config.tensor_parallel_size
|
||||
hidden_size = self.model_config.get_hidden_size()
|
||||
assert isinstance(self.model_config.dtype, torch.dtype)
|
||||
element_size = self.model_config.dtype.itemsize
|
||||
pass_config.sp_min_token_num = get_sequence_parallelism_threshold(
|
||||
hidden_size, tp_size, element_size
|
||||
)
|
||||
|
||||
if pass_config.sp_min_token_num is None:
|
||||
logger.warning_once(
|
||||
"Model hidden_size too small for the SP "
|
||||
"threshold heuristic, disabling. To force SP, "
|
||||
"set pass_config.sp_min_token_num manually."
|
||||
)
|
||||
pass_config.enable_sp = False
|
||||
pass_config.fuse_gemm_comms = False
|
||||
|
||||
from vllm.utils.torch_utils import HAS_OPAQUE_TYPE
|
||||
|
||||
if HAS_OPAQUE_TYPE:
|
||||
@@ -2184,30 +2119,6 @@ class VllmConfig:
|
||||
data_parallel_size=effective_dp_size,
|
||||
)
|
||||
|
||||
if self.compilation_config.pass_config.enable_sp:
|
||||
# With pipeline parallelism, native rms norm tracing errors due to
|
||||
# incorrect residual shape.
|
||||
# Use custom rms norm to unblock. In the future,
|
||||
# the pass will operate on higher-level IR to avoid the issue.
|
||||
# TODO: https://github.com/vllm-project/vllm/issues/27894
|
||||
if self.compilation_config.mode != CompilationMode.VLLM_COMPILE:
|
||||
logger.warning_once(
|
||||
"Sequence parallelism is enabled, but running in wrong "
|
||||
"vllm compile mode: %s.",
|
||||
self.compilation_config.mode,
|
||||
)
|
||||
|
||||
if self.parallel_config.pipeline_parallel_size > 1:
|
||||
if "-rms_norm" not in self.compilation_config.custom_ops:
|
||||
self.compilation_config.custom_ops.append("+rms_norm")
|
||||
else:
|
||||
logger.warning_once(
|
||||
"Sequence parallelism not supported with "
|
||||
"native rms_norm when using %s, "
|
||||
"this will likely lead to an error.",
|
||||
"pipeline parallelism",
|
||||
)
|
||||
|
||||
# final check of cudagraph mode after all possible updates
|
||||
if current_platform.is_cuda_alike():
|
||||
if (
|
||||
@@ -2395,29 +2306,6 @@ class VllmConfig:
|
||||
# Log the custom passes that are enabled
|
||||
self.compilation_config.pass_config.log_enabled_passes()
|
||||
|
||||
def update_sizes_for_sequence_parallelism(self, possible_sizes: list) -> list:
|
||||
# remove the sizes that not multiple of tp_size when
|
||||
# enable sequence parallelism
|
||||
removed_sizes = [
|
||||
size
|
||||
for size in possible_sizes
|
||||
if size % self.parallel_config.tensor_parallel_size != 0
|
||||
]
|
||||
if removed_sizes:
|
||||
logger.warning(
|
||||
"Batch sizes %s are removed because they are not "
|
||||
"multiple of tp_size %d when "
|
||||
"sequence parallelism is enabled",
|
||||
removed_sizes,
|
||||
self.parallel_config.tensor_parallel_size,
|
||||
)
|
||||
|
||||
return [
|
||||
size
|
||||
for size in possible_sizes
|
||||
if size % self.parallel_config.tensor_parallel_size == 0
|
||||
]
|
||||
|
||||
def _set_max_num_scheduled_tokens(self):
|
||||
"""In most cases, the scheduler may schedule a batch with as many tokens as the
|
||||
worker is configured to handle.
|
||||
@@ -2665,16 +2553,6 @@ class VllmConfig:
|
||||
# de-duplicate and sort the sizes
|
||||
cudagraph_capture_sizes = sorted(set(cudagraph_capture_sizes))
|
||||
|
||||
if (
|
||||
self.parallel_config.tensor_parallel_size > 1
|
||||
and self.compilation_config.pass_config.enable_sp
|
||||
):
|
||||
# Sequence parallelism only captures TP-divisible sizes, so a
|
||||
# wider non-divisible decode batch cannot be captured under SP.
|
||||
cudagraph_capture_sizes = self.update_sizes_for_sequence_parallelism(
|
||||
cudagraph_capture_sizes
|
||||
)
|
||||
|
||||
# user-specific compilation_config.max_cudagraph_capture_size get
|
||||
# truncated to valid_max_size when they are inconsistent.
|
||||
valid_max_size = (
|
||||
@@ -2764,37 +2642,6 @@ class VllmConfig:
|
||||
"allreduce-rms fusion will be enabled for all num_tokens."
|
||||
)
|
||||
|
||||
# Add the compile ranges for sequence parallelism
|
||||
if compilation_config.pass_config.enable_sp:
|
||||
pass_config = compilation_config.pass_config
|
||||
|
||||
# Calculate min_token_num if not explicitly provided
|
||||
# User override works regardless of hidden_size
|
||||
if pass_config.sp_min_token_num is None:
|
||||
from vllm.compilation.passes.fusion.sequence_parallelism import (
|
||||
get_sequence_parallelism_threshold,
|
||||
)
|
||||
|
||||
tp_size = self.parallel_config.tensor_parallel_size
|
||||
hidden_size = self.model_config.get_hidden_size()
|
||||
assert isinstance(self.model_config.dtype, torch.dtype)
|
||||
element_size = self.model_config.dtype.itemsize
|
||||
pass_config.sp_min_token_num = get_sequence_parallelism_threshold(
|
||||
hidden_size, tp_size, element_size
|
||||
)
|
||||
|
||||
min_token_num = pass_config.sp_min_token_num
|
||||
max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens
|
||||
if min_token_num is not None and (
|
||||
max_num_batched_tokens is not None
|
||||
and min_token_num < max_num_batched_tokens
|
||||
and min_token_num > 1
|
||||
):
|
||||
# Add endpoint at min_token_num - 1 to ensure SP applies
|
||||
# starting from min_token_num
|
||||
# This creates ranges: [1, min-1] (no SP), [min, max] (SP applies)
|
||||
computed_compile_ranges_endpoints.append(min_token_num - 1)
|
||||
|
||||
if compilation_config.pass_config.fuse_rope_kvcache:
|
||||
max_token_num = (
|
||||
compilation_config.pass_config.rope_kvcache_fusion_max_token_num
|
||||
@@ -3124,12 +2971,6 @@ class VllmConfig:
|
||||
unsupported: list[str] = []
|
||||
speculative_config = self.speculative_config
|
||||
|
||||
if (
|
||||
self.compilation_config.pass_config.enable_sp
|
||||
and self.parallel_config.tensor_parallel_size > 1
|
||||
):
|
||||
unsupported.append("sequence parallelism")
|
||||
|
||||
# V2 does not implement the external_launcher (torchrun) PP-output
|
||||
# broadcast that V1 uses to keep all ranks in sync (broadcast_pp_output).
|
||||
if (
|
||||
|
||||
Reference in New Issue
Block a user