[Bugfix] Keep JIT warmup under enforce-eager when fault tolerance is on (#58593)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Kimi <noreply@moonshot.cn>
This commit is contained in:
@@ -264,6 +264,21 @@ def test_enforce_eager(vllm_runner, monkeypatch):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enable_fault_tolerance", [False, True])
|
||||
def test_enforce_eager_jit_warmup(enable_fault_tolerance):
|
||||
"""Enforce-eager disables JIT warmup unless fault tolerance is on.
|
||||
|
||||
FT fault detection runs against deadlines that in-inference Triton
|
||||
compilation latency spikes can blow past, so warmup stays enabled.
|
||||
"""
|
||||
config = VllmConfig(
|
||||
model_config=ModelConfig(model="facebook/opt-125m", enforce_eager=True),
|
||||
parallel_config=ParallelConfig(enable_fault_tolerance=enable_fault_tolerance),
|
||||
)
|
||||
assert config.compilation_config.mode == CompilationMode.NONE
|
||||
assert config.kernel_config.enable_jit_warmup == enable_fault_tolerance
|
||||
|
||||
|
||||
@pytest.mark.forked
|
||||
def test_torch_compile_disable(vllm_runner, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
|
||||
+16
-7
@@ -1666,15 +1666,24 @@ class VllmConfig:
|
||||
)
|
||||
|
||||
if self.model_config is not None and self.model_config.enforce_eager:
|
||||
logger.warning_once(
|
||||
"Enforce eager set, disabling torch.compile, CUDAGraphs, and JIT "
|
||||
"kernel warmup. This is equivalent to setting -cc.mode=none "
|
||||
"-cc.cudagraph_mode=none and "
|
||||
"--kernel_config.enable_jit_warmup=False"
|
||||
)
|
||||
self.compilation_config.mode = CompilationMode.NONE
|
||||
self.compilation_config.cudagraph_mode = CUDAGraphMode.NONE
|
||||
self.kernel_config.enable_jit_warmup = False
|
||||
if self.parallel_config.enable_fault_tolerance:
|
||||
# Keep JIT warmup: in-inference Triton compilation latency
|
||||
# spikes can delay peer-fault detection past its deadline.
|
||||
logger.warning_once(
|
||||
"Enforce eager set, disabling torch.compile and CUDAGraphs. "
|
||||
"This is equivalent to setting -cc.mode=none "
|
||||
"-cc.cudagraph_mode=none"
|
||||
)
|
||||
else:
|
||||
logger.warning_once(
|
||||
"Enforce eager set, disabling torch.compile, CUDAGraphs, and "
|
||||
"JIT kernel warmup. This is equivalent to setting "
|
||||
"-cc.mode=none -cc.cudagraph_mode=none and "
|
||||
"--kernel_config.enable_jit_warmup=False"
|
||||
)
|
||||
self.kernel_config.enable_jit_warmup = False
|
||||
|
||||
if os.environ.get("TORCH_COMPILE_DISABLE") == "1":
|
||||
logger.warning_once(
|
||||
|
||||
Reference in New Issue
Block a user