[torch.compile] Compile CustomOp.forward_native for SiluAndMul and QuantFP8 to avoid raw torch ops inside opaque custom ops (#32806)

Signed-off-by: Luka Govedič <lgovedic@redhat.com>
Signed-off-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
Luka Govedič
2026-01-22 19:52:26 -08:00
committed by GitHub
co-authored by Michael Goin
parent f61c9da711
commit 5e4e0e51f4
7 changed files with 52 additions and 13 deletions
+42 -7
View File
@@ -1,16 +1,15 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import torch.nn as nn
from vllm.config import get_cached_compilation_config
from vllm.logger import init_logger
from vllm.model_executor.utils import maybe_disable_graph_partition
from vllm.platforms import current_platform
logger = init_logger(__name__)
# Dictionary of all custom ops (classes, indexed by registered name).
# To check if an op with a name is enabled, call .enabled() on the class.
# Examples:
@@ -118,10 +117,10 @@ class CustomOp(nn.Module):
)
return super().__new__(op_cls_to_instantiate)
def __init__(self, enforce_enable: bool = False):
def __init__(self, *, enforce_enable: bool = False, compile_native: bool = False):
super().__init__()
self._enforce_enable = enforce_enable
self._forward_method = self.dispatch_forward()
self._forward_method = self.dispatch_forward(compile_native=compile_native)
def forward(self, *args, **kwargs):
return self._forward_method(*args, **kwargs)
@@ -162,7 +161,7 @@ class CustomOp(nn.Module):
# PyTorch-native implementation.
return self.forward_native(*args, **kwargs)
def dispatch_forward(self):
def dispatch_forward(self, compile_native: bool):
# NOTE(woosuk): Here we assume that vLLM was built for only one
# specific backend. Currently, we do not support dynamic dispatching.
compilation_config = get_cached_compilation_config()
@@ -180,7 +179,9 @@ class CustomOp(nn.Module):
compilation_config.disabled_custom_ops.update([self.__class__.name])
if not enabled:
return self.forward_native
# Compile forward_native to avoid eager torch ops if inside
# opaque torch custom op (e.g. fused_moe, unified_attention, etc.)
return self.maybe_compile(self.forward_native, enable=compile_native)
if current_platform.is_rocm():
return self.forward_hip
@@ -195,6 +196,40 @@ class CustomOp(nn.Module):
else:
return self.forward_cuda
def maybe_compile(self, fn, *, enable: bool = True):
"""
Compile fn if compilation enabled.
Useful for CustomOp instances called from within a torch custom op,
meaning the forward call is hidden from the model-level torch.compile.
NOTE: this does not enable fusion across ops, so opaque custom ops
should still be unwrapped wherever possible.
"""
# Do not compile if compilation disabled
from vllm.config.compilation import CompilationMode
if not enable:
return fn
# Do not compile if global compilation disabled
compilation_config = get_cached_compilation_config()
if compilation_config.mode == CompilationMode.NONE:
return fn
# If eager backend is used, do not compile either
if compilation_config.backend == "eager":
return fn
# dynamic=True to avoid recompilations
return torch.compile(
fn,
dynamic=True,
backend=current_platform.simple_compile_backend,
options=maybe_disable_graph_partition(
current_platform.simple_compile_backend
),
)
@classmethod
def enabled(cls) -> bool:
# if no name, then it was not registered