[Bugfix] Raise for unavailable piecewise CUDA graphs (#54782)
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -18,6 +18,7 @@ from vllm.compilation.breakable_cudagraph import (
|
||||
)
|
||||
from vllm.compilation.counter import compilation_counter
|
||||
from vllm.compilation.cuda_graph import CUDAGraphWrapper
|
||||
from vllm.compilation.wrapper import TorchCompileWithNoGuardsWrapper
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id
|
||||
@@ -108,6 +109,15 @@ def _is_compatible(
|
||||
)
|
||||
|
||||
|
||||
def has_compiled_submodule(model: nn.Module) -> bool:
|
||||
"""Whether any submodule is an active @support_torch_compile module."""
|
||||
return any(
|
||||
isinstance(m, TorchCompileWithNoGuardsWrapper)
|
||||
and not getattr(m, "do_not_compile", True)
|
||||
for m in model.modules()
|
||||
)
|
||||
|
||||
|
||||
class CudaGraphManager:
|
||||
def __init__(
|
||||
self,
|
||||
@@ -510,6 +520,16 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
if self.use_breakable_cg:
|
||||
self.init_breakable_cg_runner(model)
|
||||
|
||||
if self.cudagraph_mode.has_piecewise_cudagraphs() and not (
|
||||
self.use_breakable_cg or has_compiled_submodule(model)
|
||||
):
|
||||
raise RuntimeError(
|
||||
f"{type(model).__name__}: piecewise CUDA graphs "
|
||||
f"(cudagraph_mode={self.cudagraph_mode.name}) unavailable, "
|
||||
"model is not torch-compiled and breakable CUDA graph is off. "
|
||||
"Set VLLM_USE_BREAKABLE_CUDAGRAPH=1 or cudagraph_mode=NONE/FULL."
|
||||
)
|
||||
|
||||
def create_forward_fn(
|
||||
desc: BatchExecutionDescriptor,
|
||||
warmup: bool,
|
||||
|
||||
Reference in New Issue
Block a user