From d3e69fd6714e9d1bb6e8e4f03157090dc32e7960 Mon Sep 17 00:00:00 2001 From: Summer Yang Date: Tue, 7 Jul 2026 02:36:10 -0700 Subject: [PATCH] [Perf] Use blocking CUDA events to avoid busy polling cuda driver lock (#47081) Signed-off-by: Jingyi Yang --- vllm/v1/worker/gpu/async_utils.py | 6 ++++-- vllm/v1/worker/gpu/spec_decode/utils.py | 3 ++- vllm/v1/worker/gpu_model_runner.py | 10 +++++++--- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/vllm/v1/worker/gpu/async_utils.py b/vllm/v1/worker/gpu/async_utils.py index b3d6f5e4d9..e4659104f4 100644 --- a/vllm/v1/worker/gpu/async_utils.py +++ b/vllm/v1/worker/gpu/async_utils.py @@ -24,7 +24,8 @@ class AsyncOutput(AsyncModelRunnerOutput): self.model_runner_output = model_runner_output self.sampler_output = sampler_output self.num_sampled_tokens = num_sampled_tokens - self.copy_event = torch.cuda.Event() + # Blocking (sleep) event to avoid busy-polling the CUDA driver lock. + self.copy_event = torch.cuda.Event(blocking=True) with stream(copy_stream, main_stream): copy_stream.wait_stream(main_stream) @@ -81,7 +82,8 @@ class AsyncPoolingOutput(AsyncModelRunnerOutput): self.model_runner_output = model_runner_output self.pooler_output = pooler_output self.is_valid = is_valid - self.copy_event = torch.cuda.Event() + # Blocking (sleep) event to avoid busy-polling the CUDA driver lock. + self.copy_event = torch.cuda.Event(blocking=True) with stream(copy_stream, main_stream): copy_stream.wait_stream(main_stream) diff --git a/vllm/v1/worker/gpu/spec_decode/utils.py b/vllm/v1/worker/gpu/spec_decode/utils.py index 89d46a7582..e25672e953 100644 --- a/vllm/v1/worker/gpu/spec_decode/utils.py +++ b/vllm/v1/worker/gpu/spec_decode/utils.py @@ -12,7 +12,8 @@ class DraftTokensHandler: def __init__(self, device: torch.device | None = None): self.device = device self.copy_stream = torch.cuda.Stream(device) - self.copy_event = torch.cuda.Event() + # Blocking (sleep) event to avoid busy-polling the CUDA driver lock. + self.copy_event = torch.cuda.Event(blocking=True) self.req_ids: list[str] = [] self.draft_tokens_np: np.ndarray | None = None diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 24889f4021..5cf3f5728b 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -259,7 +259,8 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput): self._invalid_req_indices = invalid_req_indices # Event on the copy stream so we can synchronize the non-blocking copy. - self.async_copy_ready_event = torch.Event() + # Blocking (sleep) event to avoid busy-polling the CUDA driver lock. + self.async_copy_ready_event = torch.cuda.Event(blocking=True) # Keep a reference to the device tensor to avoid it being # deallocated until we finish copying it to the host. @@ -392,7 +393,8 @@ class AsyncGPUPoolingModelRunnerOutput(AsyncModelRunnerOutput): self._model_runner_output = model_runner_output # Event on the copy stream so we can synchronize the non-blocking copy. - self.async_copy_ready_event = torch.Event() + # Blocking (sleep) event to avoid busy-polling the CUDA driver lock. + self.async_copy_ready_event = torch.cuda.Event(blocking=True) # Keep a reference to the device tensors to avoid them being # deallocated until we finish copying it to the host. @@ -718,7 +720,9 @@ class GPUModelRunner( self.prepare_inputs_event: torch.Event | None = None if self.use_async_scheduling: self.async_output_copy_stream = torch.cuda.Stream() - self.prepare_inputs_event = torch.Event() + # Blocking (sleep) event to avoid busy-polling the CUDA driver lock; + # under TP contention that spin can balloon and make the rank a straggler. + self.prepare_inputs_event = torch.cuda.Event(blocking=True) # self.cudagraph_batch_sizes sorts in ascending order. if (