[ROCm][Bugfix] Reduce CUDA graph divergences (#57229)
Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
This commit is contained in:
@@ -98,6 +98,7 @@ from vllm.v1.worker.worker_base import CompilationTimes, WorkerBase
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
from ...model_executor.model_loader import TensorizerLoader
|
||||
from .gpu.cudagraph_utils import has_compiled_submodule
|
||||
from .gpu.warmup import warmup_kernels
|
||||
from .utils import request_memory
|
||||
|
||||
@@ -772,7 +773,10 @@ class Worker(WorkerBase):
|
||||
def compile_or_warm_up_model(self) -> CompilationTimes:
|
||||
warmup_sizes: list[int] = []
|
||||
|
||||
if self.vllm_config.compilation_config.mode == CompilationMode.VLLM_COMPILE:
|
||||
if (
|
||||
self.vllm_config.compilation_config.mode == CompilationMode.VLLM_COMPILE
|
||||
and has_compiled_submodule(self.model_runner.get_model())
|
||||
):
|
||||
# warm up sizes that are not in cudagraph capture sizes,
|
||||
# but users still want to compile for better performance,
|
||||
# e.g. for the max-num-batched token size in chunked prefill.
|
||||
|
||||
Reference in New Issue
Block a user