Signed-off-by: arpitkh101 <arpit5khandelwal@gmail.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
This commit is contained in:
co-authored by
Luka Govedič
parent
506ed87e87
commit
d7284a2604
@@ -92,22 +92,23 @@ class PostGradPassManager(CustomGraphPass):
|
||||
|
||||
# Set the current vllm config to allow tracing CustomOp instances
|
||||
with set_current_vllm_config(config, check_compile=False):
|
||||
if self.pass_config.enable_noop:
|
||||
if self.pass_config.eliminate_noops:
|
||||
self.passes += [NoOpEliminationPass(config)]
|
||||
|
||||
if self.pass_config.enable_sequence_parallelism:
|
||||
if self.pass_config.enable_sp:
|
||||
self.passes += [SequenceParallelismPass(config)]
|
||||
if self.pass_config.enable_async_tp:
|
||||
if self.pass_config.fuse_gemm_comms:
|
||||
self.passes += [AsyncTPPass(config)]
|
||||
|
||||
if self.pass_config.enable_fi_allreduce_fusion:
|
||||
if self.pass_config.fuse_allreduce_rms:
|
||||
self.passes += [AllReduceFusionPass(config)]
|
||||
|
||||
if self.pass_config.enable_fusion:
|
||||
if self.pass_config.fuse_norm_quant:
|
||||
self.passes += [RMSNormQuantFusionPass(config)]
|
||||
if self.pass_config.fuse_act_quant:
|
||||
self.passes += [ActivationQuantFusionPass(config)]
|
||||
|
||||
if self.pass_config.enable_attn_fusion:
|
||||
if self.pass_config.fuse_attn_quant:
|
||||
self.passes += [AttnFusionPass(config)]
|
||||
|
||||
if self.pass_config.enable_qk_norm_rope_fusion:
|
||||
|
||||
+102
-18
@@ -13,7 +13,7 @@ from pydantic.dataclasses import dataclass
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.compilation.inductor_pass import CallableInductorPass, InductorPass
|
||||
from vllm.config.utils import config
|
||||
from vllm.config.utils import config, handle_deprecated
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import resolve_obj_by_qualname
|
||||
@@ -105,18 +105,43 @@ class PassConfig:
|
||||
improper state.
|
||||
"""
|
||||
|
||||
# New flags
|
||||
fuse_norm_quant: bool = Field(default=None)
|
||||
"""Fuse the custom RMSNorm + quant ops."""
|
||||
fuse_act_quant: bool = Field(default=None)
|
||||
"""Fuse the custom SiluMul + quant ops."""
|
||||
fuse_attn_quant: bool = Field(default=None)
|
||||
"""Fuse the custom attention + quant ops."""
|
||||
eliminate_noops: bool = Field(default=None)
|
||||
"""Eliminate no-op ops."""
|
||||
enable_sp: bool = Field(default=None)
|
||||
"""Enable sequence parallelism."""
|
||||
fuse_gemm_comms: bool = Field(default=None)
|
||||
"""Enable async TP."""
|
||||
fuse_allreduce_rms: bool = Field(default=None)
|
||||
"""Enable flashinfer allreduce fusion."""
|
||||
|
||||
# Deprecated flags
|
||||
enable_fusion: bool = Field(default=None)
|
||||
"""Whether to enable the custom fusion (RMSNorm/SiluMul+quant) pass."""
|
||||
"""Deprecated in: v0.12.0. Use fuse_norm_quant and fuse_act_quant
|
||||
instead. Will be removed in v0.13.0 or v1.0.0, whichever is sooner.
|
||||
"""
|
||||
enable_attn_fusion: bool = Field(default=None)
|
||||
"""Whether to enable the custom attention+quant fusion pass."""
|
||||
"""Deprecated in: v0.12.0. Use fuse_attn_quant instead.
|
||||
Will be removed in v0.13.0 or v1.0.0, whichever is sooner."""
|
||||
enable_noop: bool = Field(default=None)
|
||||
"""Whether to enable the custom no-op elimination pass."""
|
||||
"""Deprecated in: v0.12.0. Use eliminate_noops instead.
|
||||
Will be removed in v0.13.0 or v1.0.0, whichever is sooner."""
|
||||
enable_sequence_parallelism: bool = Field(default=None)
|
||||
"""Whether to enable sequence parallelism."""
|
||||
"""Deprecated in: v0.12.0. Use enable_sp instead.
|
||||
Will be removed in v0.13.0 or v1.0.0, whichever is sooner."""
|
||||
enable_async_tp: bool = Field(default=None)
|
||||
"""Whether to enable async TP."""
|
||||
"""Deprecated in: v0.12.0. Use fuse_gemm_comms instead.
|
||||
Will be removed in v0.13.0 or v1.0.0, whichever is sooner."""
|
||||
enable_fi_allreduce_fusion: bool = Field(default=None)
|
||||
"""Whether to enable flashinfer allreduce fusion."""
|
||||
"""Deprecated in: v0.12.0. Use fuse_allreduce_rms instead.
|
||||
Will be removed in v0.13.0 or v1.0.0, whichever is sooner."""
|
||||
|
||||
fi_allreduce_fusion_max_size_mb: float | None = None
|
||||
"""The threshold of the communicated tensor sizes under which
|
||||
vllm should use flashinfer fused allreduce. Specified as a
|
||||
@@ -136,7 +161,7 @@ class PassConfig:
|
||||
},
|
||||
}, where key is the device capability"""
|
||||
enable_qk_norm_rope_fusion: bool = False
|
||||
"""Whether to enable the fused Q/K RMSNorm + RoPE pass."""
|
||||
"""Enable fused Q/K RMSNorm + RoPE pass."""
|
||||
|
||||
# TODO(luka) better pass enabling system.
|
||||
|
||||
@@ -174,6 +199,13 @@ class PassConfig:
|
||||
return InductorPass.hash_dict(asdict(self))
|
||||
|
||||
@field_validator(
|
||||
"fuse_norm_quant",
|
||||
"fuse_act_quant",
|
||||
"fuse_attn_quant",
|
||||
"eliminate_noops",
|
||||
"enable_sp",
|
||||
"fuse_gemm_comms",
|
||||
"fuse_allreduce_rms",
|
||||
"enable_fusion",
|
||||
"enable_attn_fusion",
|
||||
"enable_noop",
|
||||
@@ -190,18 +222,71 @@ class PassConfig:
|
||||
return handler(value)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.enable_noop:
|
||||
if self.enable_fusion:
|
||||
# Handle deprecation and defaults
|
||||
|
||||
# Map old flags to new flags and issue warnings
|
||||
handle_deprecated(
|
||||
self,
|
||||
"enable_fusion",
|
||||
["fuse_norm_quant", "fuse_act_quant"],
|
||||
"v0.13.0 or v1.0.0, whichever is sooner",
|
||||
)
|
||||
|
||||
handle_deprecated(
|
||||
self,
|
||||
"enable_attn_fusion",
|
||||
"fuse_attn_quant",
|
||||
"v0.13.0 or v1.0.0, whichever is sooner",
|
||||
)
|
||||
|
||||
handle_deprecated(
|
||||
self,
|
||||
"enable_sequence_parallelism",
|
||||
"enable_sp",
|
||||
"v0.13.0 or v1.0.0, whichever is sooner",
|
||||
)
|
||||
|
||||
handle_deprecated(
|
||||
self,
|
||||
"enable_async_tp",
|
||||
"fuse_gemm_comms",
|
||||
"v0.13.0 or v1.0.0, whichever is sooner",
|
||||
)
|
||||
|
||||
handle_deprecated(
|
||||
self,
|
||||
"enable_fi_allreduce_fusion",
|
||||
"fuse_allreduce_rms",
|
||||
"v0.13.0 or v1.0.0, whichever is sooner",
|
||||
)
|
||||
|
||||
handle_deprecated(
|
||||
self,
|
||||
"enable_noop",
|
||||
"eliminate_noops",
|
||||
"v0.13.0 or v1.0.0, whichever is sooner",
|
||||
)
|
||||
|
||||
# Force old flags to None to ensure they are not used
|
||||
self.enable_fusion = None
|
||||
self.enable_attn_fusion = None
|
||||
self.enable_noop = None
|
||||
self.enable_sequence_parallelism = None
|
||||
self.enable_async_tp = None
|
||||
self.enable_fi_allreduce_fusion = None
|
||||
|
||||
if not self.eliminate_noops:
|
||||
if self.fuse_norm_quant or self.fuse_act_quant:
|
||||
logger.warning_once(
|
||||
"Fusion enabled but reshape elimination disabled. "
|
||||
"RMSNorm/SiluMul + quant (fp8) fusion might not work"
|
||||
)
|
||||
if self.enable_attn_fusion:
|
||||
if self.fuse_attn_quant:
|
||||
logger.warning_once(
|
||||
"Fusion enabled but reshape elimination disabled. "
|
||||
"Attention + quant (fp8) fusion might not work"
|
||||
)
|
||||
if self.enable_fi_allreduce_fusion:
|
||||
if self.fuse_allreduce_rms:
|
||||
logger.warning_once(
|
||||
"Fusion enabled but reshape elimination disabled. "
|
||||
"Allreduce + rms norm + quant (fp8) fusion might not work"
|
||||
@@ -873,7 +958,7 @@ class CompilationConfig:
|
||||
self.set_splitting_ops_for_inductor_graph_partition()
|
||||
return
|
||||
|
||||
if self.pass_config.enable_attn_fusion:
|
||||
if self.pass_config.fuse_attn_quant:
|
||||
# here use_inductor_graph_partition is False
|
||||
self.set_splitting_ops_for_attn_fusion()
|
||||
return
|
||||
@@ -915,12 +1000,12 @@ class CompilationConfig:
|
||||
self.splitting_ops = list(self._attention_ops)
|
||||
|
||||
def set_splitting_ops_for_attn_fusion(self):
|
||||
assert self.pass_config.enable_attn_fusion
|
||||
assert self.pass_config.fuse_attn_quant
|
||||
if self.splitting_ops is None:
|
||||
self.splitting_ops = []
|
||||
if self.cudagraph_mode.has_piecewise_cudagraphs():
|
||||
logger.warning_once(
|
||||
"enable_attn_fusion is incompatible with piecewise "
|
||||
"fuse_attn_quant is incompatible with piecewise "
|
||||
"cudagraph when use_inductor_graph_partition is off. "
|
||||
"In this case, splitting_ops will be set to empty "
|
||||
"list, and cudagraph_mode will be set to FULL. "
|
||||
@@ -931,8 +1016,7 @@ class CompilationConfig:
|
||||
self.cudagraph_mode = CUDAGraphMode.FULL
|
||||
|
||||
assert not self.splitting_ops_contain_attention(), (
|
||||
"attention ops should not be in splitting_ops "
|
||||
"when enable_attn_fusion is True"
|
||||
"attention ops should not be in splitting_ops when fuse_attn_quant is True"
|
||||
)
|
||||
|
||||
def splitting_ops_contain_attention(self) -> bool:
|
||||
@@ -1008,7 +1092,7 @@ class CompilationConfig:
|
||||
self, uniform_decode_query_len: int, tensor_parallel_size: int
|
||||
):
|
||||
multiple_of = uniform_decode_query_len
|
||||
if tensor_parallel_size > 1 and self.pass_config.enable_sequence_parallelism:
|
||||
if tensor_parallel_size > 1 and self.pass_config.enable_sp:
|
||||
multiple_of = max(uniform_decode_query_len, tensor_parallel_size)
|
||||
if (
|
||||
multiple_of % uniform_decode_query_len != 0
|
||||
|
||||
@@ -19,6 +19,10 @@ import torch
|
||||
from pydantic.fields import FieldInfo
|
||||
from typing_extensions import runtime_checkable
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from _typeshed import DataclassInstance
|
||||
else:
|
||||
@@ -293,3 +297,28 @@ def get_hash_factors(config: ConfigT, ignored_factors: set[str]) -> dict[str, ob
|
||||
def hash_factors(items: dict[str, object]) -> str:
|
||||
"""Return a SHA-256 hex digest of the canonical items structure."""
|
||||
return hashlib.sha256(json.dumps(items, sort_keys=True).encode()).hexdigest()
|
||||
|
||||
|
||||
def handle_deprecated(
|
||||
config: ConfigT,
|
||||
old_name: str,
|
||||
new_name_or_names: str | list[str],
|
||||
removal_version: str,
|
||||
) -> None:
|
||||
old_val = getattr(config, old_name)
|
||||
if old_val is None:
|
||||
return
|
||||
|
||||
if isinstance(new_name_or_names, str):
|
||||
new_names = [new_name_or_names]
|
||||
else:
|
||||
new_names = new_name_or_names
|
||||
|
||||
msg = (
|
||||
f"{old_name} is deprecated and will be removed in {removal_version}. "
|
||||
f"Use {', '.join(new_names)} instead."
|
||||
)
|
||||
logger.warning(msg)
|
||||
|
||||
for new_name in new_names:
|
||||
setattr(config, new_name, old_val)
|
||||
|
||||
+45
-31
@@ -83,22 +83,33 @@ IS_DENSE = False
|
||||
# See https://github.com/vllm-project/vllm/issues/25689.
|
||||
|
||||
|
||||
def enable_fusion(cfg: "VllmConfig") -> bool:
|
||||
"""Returns True if RMS norm or quant FP8 is enabled."""
|
||||
def enable_norm_fusion(cfg: "VllmConfig") -> bool:
|
||||
"""Enable if either RMS norm or quant FP8 custom op is active;
|
||||
otherwise Inductor handles fusion."""
|
||||
|
||||
return cfg.compilation_config.is_custom_op_enabled(
|
||||
"rms_norm"
|
||||
) or cfg.compilation_config.is_custom_op_enabled("quant_fp8")
|
||||
|
||||
|
||||
def enable_act_fusion(cfg: "VllmConfig") -> bool:
|
||||
"""Enable if either SiLU+Mul or quant FP8 custom op is active;
|
||||
otherwise Inductor handles fusion."""
|
||||
return cfg.compilation_config.is_custom_op_enabled(
|
||||
"silu_and_mul"
|
||||
) or cfg.compilation_config.is_custom_op_enabled("quant_fp8")
|
||||
|
||||
|
||||
OPTIMIZATION_LEVEL_00 = {
|
||||
"compilation_config": {
|
||||
"pass_config": {
|
||||
"enable_noop": False,
|
||||
"enable_fusion": False,
|
||||
"enable_fi_allreduce_fusion": False,
|
||||
"enable_attn_fusion": False,
|
||||
"enable_sequence_parallelism": False,
|
||||
"enable_async_tp": False,
|
||||
"eliminate_noops": False,
|
||||
"fuse_norm_quant": False,
|
||||
"fuse_act_quant": False,
|
||||
"fuse_allreduce_rms": False,
|
||||
"fuse_attn_quant": False,
|
||||
"enable_sp": False,
|
||||
"fuse_gemm_comms": False,
|
||||
},
|
||||
"cudagraph_mode": CUDAGraphMode.NONE,
|
||||
"use_inductor_graph_partition": False,
|
||||
@@ -107,12 +118,13 @@ OPTIMIZATION_LEVEL_00 = {
|
||||
OPTIMIZATION_LEVEL_01 = {
|
||||
"compilation_config": {
|
||||
"pass_config": {
|
||||
"enable_noop": True,
|
||||
"enable_fusion": enable_fusion,
|
||||
"enable_fi_allreduce_fusion": False,
|
||||
"enable_attn_fusion": False,
|
||||
"enable_sequence_parallelism": False,
|
||||
"enable_async_tp": False,
|
||||
"eliminate_noops": True,
|
||||
"fuse_norm_quant": enable_norm_fusion,
|
||||
"fuse_act_quant": enable_act_fusion,
|
||||
"fuse_allreduce_rms": False,
|
||||
"fuse_attn_quant": False,
|
||||
"enable_sp": False,
|
||||
"fuse_gemm_comms": False,
|
||||
},
|
||||
"cudagraph_mode": CUDAGraphMode.PIECEWISE,
|
||||
"use_inductor_graph_partition": False,
|
||||
@@ -121,12 +133,13 @@ OPTIMIZATION_LEVEL_01 = {
|
||||
OPTIMIZATION_LEVEL_02 = {
|
||||
"compilation_config": {
|
||||
"pass_config": {
|
||||
"enable_noop": True,
|
||||
"enable_fusion": enable_fusion,
|
||||
"enable_fi_allreduce_fusion": False,
|
||||
"enable_attn_fusion": IS_QUANTIZED,
|
||||
"enable_sequence_parallelism": IS_DENSE,
|
||||
"enable_async_tp": IS_DENSE,
|
||||
"eliminate_noops": True,
|
||||
"fuse_norm_quant": enable_norm_fusion,
|
||||
"fuse_act_quant": enable_act_fusion,
|
||||
"fuse_allreduce_rms": False,
|
||||
"fuse_attn_quant": IS_QUANTIZED,
|
||||
"enable_sp": IS_DENSE,
|
||||
"fuse_gemm_comms": IS_DENSE,
|
||||
},
|
||||
"cudagraph_mode": CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
"use_inductor_graph_partition": False,
|
||||
@@ -135,12 +148,13 @@ OPTIMIZATION_LEVEL_02 = {
|
||||
OPTIMIZATION_LEVEL_03 = {
|
||||
"compilation_config": {
|
||||
"pass_config": {
|
||||
"enable_noop": True,
|
||||
"enable_fusion": enable_fusion,
|
||||
"enable_fi_allreduce_fusion": False,
|
||||
"enable_attn_fusion": IS_QUANTIZED,
|
||||
"enable_sequence_parallelism": IS_DENSE,
|
||||
"enable_async_tp": IS_DENSE,
|
||||
"eliminate_noops": True,
|
||||
"fuse_norm_quant": enable_norm_fusion,
|
||||
"fuse_act_quant": enable_act_fusion,
|
||||
"fuse_allreduce_rms": False,
|
||||
"fuse_attn_quant": IS_QUANTIZED,
|
||||
"enable_sp": IS_DENSE,
|
||||
"fuse_gemm_comms": IS_DENSE,
|
||||
},
|
||||
"cudagraph_mode": CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
"use_inductor_graph_partition": False,
|
||||
@@ -645,9 +659,9 @@ class VllmConfig:
|
||||
|
||||
# async tp is built on top of sequence parallelism
|
||||
# and requires it to be enabled.
|
||||
if self.compilation_config.pass_config.enable_async_tp:
|
||||
self.compilation_config.pass_config.enable_sequence_parallelism = True
|
||||
if self.compilation_config.pass_config.enable_sequence_parallelism:
|
||||
if self.compilation_config.pass_config.fuse_gemm_comms:
|
||||
self.compilation_config.pass_config.enable_sp = True
|
||||
if self.compilation_config.pass_config.enable_sp:
|
||||
if "-rms_norm" in self.compilation_config.custom_ops:
|
||||
logger.warning(
|
||||
"RMS norm force disabled, sequence parallelism might break"
|
||||
@@ -797,7 +811,7 @@ class VllmConfig:
|
||||
# Do this after all the updates to compilation_config.mode
|
||||
self.compilation_config.set_splitting_ops_for_v1()
|
||||
|
||||
if self.compilation_config.pass_config.enable_sequence_parallelism:
|
||||
if self.compilation_config.pass_config.enable_sp:
|
||||
# With pipeline parallelism or dynamo partitioning,
|
||||
# native rms norm tracing errors due to incorrect residual shape.
|
||||
# Use custom rms norm to unblock. In the future,
|
||||
@@ -1062,7 +1076,7 @@ class VllmConfig:
|
||||
|
||||
if (
|
||||
self.parallel_config.tensor_parallel_size > 1
|
||||
and self.compilation_config.pass_config.enable_sequence_parallelism
|
||||
and self.compilation_config.pass_config.enable_sp
|
||||
):
|
||||
cudagraph_capture_sizes = self.update_sizes_for_sequence_parallelism(
|
||||
cudagraph_capture_sizes
|
||||
|
||||
@@ -2417,10 +2417,7 @@ class GPUModelRunner(
|
||||
# Pad tokens to multiple of tensor_parallel_size when
|
||||
# enabled collective fusion for SP
|
||||
tp_size = self.vllm_config.parallel_config.tensor_parallel_size
|
||||
if (
|
||||
self.compilation_config.pass_config.enable_sequence_parallelism
|
||||
and tp_size > 1
|
||||
):
|
||||
if self.compilation_config.pass_config.enable_sp and tp_size > 1:
|
||||
return round_up(num_scheduled_tokens, tp_size)
|
||||
return num_scheduled_tokens
|
||||
|
||||
|
||||
@@ -552,7 +552,7 @@ class Worker(WorkerBase):
|
||||
|
||||
if (
|
||||
parallel_config.pipeline_parallel_size > 1
|
||||
and compilation_config.pass_config.enable_sequence_parallelism
|
||||
and compilation_config.pass_config.enable_sp
|
||||
and forward_pass
|
||||
):
|
||||
# currently only supported by V1 GPUModelRunner
|
||||
|
||||
@@ -342,7 +342,7 @@ def is_residual_scattered_for_sp(
|
||||
partition), SP is always applied
|
||||
- Otherwise, SP is only applied for specific shapes in compile_sizes
|
||||
"""
|
||||
if not vllm_config.compilation_config.pass_config.enable_sequence_parallelism:
|
||||
if not vllm_config.compilation_config.pass_config.enable_sp:
|
||||
return False
|
||||
|
||||
tp = vllm_config.parallel_config.tensor_parallel_size
|
||||
|
||||
Reference in New Issue
Block a user