[Core] Add VLLM_GPU_SYNC_CHECK env var (#44800)
This commit is contained in:
@@ -56,6 +56,7 @@ from vllm.sequence import IntermediateTensors
|
||||
from vllm.tasks import SupportedTask
|
||||
from vllm.tracing import instrument
|
||||
from vllm.utils.gc_utils import freeze_gc_heap, maybe_attach_gc_debug_callback
|
||||
from vllm.utils.gpu_sync_debug import enable_gpu_sync_check, with_gpu_sync_check
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
from vllm.utils.mem_utils import MemorySnapshot, format_gib, memory_profiling
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
@@ -759,6 +760,16 @@ class Worker(WorkerBase):
|
||||
# the model initialization and profiling.
|
||||
set_random_seed(self.model_config.seed)
|
||||
|
||||
# Eagerly trigger inductor's once-per-process lazy inits during
|
||||
# warmup (rather than on a later compile cache-miss at runtime).
|
||||
c_config = self.compilation_config
|
||||
if c_config.mode != CompilationMode.NONE and c_config.backend == "inductor":
|
||||
from vllm.compilation.compiler_interface import (
|
||||
trigger_inductor_lazy_init,
|
||||
)
|
||||
|
||||
trigger_inductor_lazy_init(self.device)
|
||||
|
||||
# All warmup is done — start monitoring for unexpected JIT
|
||||
# compilations that would cause latency spikes during inference.
|
||||
from vllm.utils.jit_monitor import activate as activate_jit_monitor
|
||||
@@ -773,6 +784,10 @@ class Worker(WorkerBase):
|
||||
freeze_gc_heap()
|
||||
maybe_attach_gc_debug_callback()
|
||||
|
||||
# Warmup / first-compile is done — activate the `VLLM_GPU_SYNC_CHECK`
|
||||
# gate so subsequent `execute_model` / `sample_tokens` calls enforce it.
|
||||
enable_gpu_sync_check()
|
||||
|
||||
return CompilationTimes(
|
||||
language_model=self.compilation_config.compilation_time,
|
||||
encoder=self.compilation_config.encoder_compilation_time,
|
||||
@@ -826,12 +841,14 @@ class Worker(WorkerBase):
|
||||
return self.profiler.annotate_context_manager(annotation)
|
||||
|
||||
@torch.inference_mode()
|
||||
@with_gpu_sync_check
|
||||
def sample_tokens(
|
||||
self, grammar_output: "GrammarOutput | None"
|
||||
) -> ModelRunnerOutput | AsyncModelRunnerOutput:
|
||||
return self.model_runner.sample_tokens(grammar_output)
|
||||
|
||||
@torch.inference_mode()
|
||||
@with_gpu_sync_check
|
||||
def execute_model(
|
||||
self, scheduler_output: "SchedulerOutput"
|
||||
) -> ModelRunnerOutput | AsyncModelRunnerOutput | None:
|
||||
|
||||
Reference in New Issue
Block a user