[Perf] Use blocking CUDA events to avoid busy polling cuda driver lock (#47081)

Signed-off-by: Jingyi Yang <girasoleyang@gmail.com>
This commit is contained in:
Summer Yang
2026-07-07 09:36:10 +00:00
committed by GitHub
parent c85d72076a
commit d3e69fd671
3 changed files with 13 additions and 6 deletions
+4 -2
View File
@@ -24,7 +24,8 @@ class AsyncOutput(AsyncModelRunnerOutput):
self.model_runner_output = model_runner_output
self.sampler_output = sampler_output
self.num_sampled_tokens = num_sampled_tokens
self.copy_event = torch.cuda.Event()
# Blocking (sleep) event to avoid busy-polling the CUDA driver lock.
self.copy_event = torch.cuda.Event(blocking=True)
with stream(copy_stream, main_stream):
copy_stream.wait_stream(main_stream)
@@ -81,7 +82,8 @@ class AsyncPoolingOutput(AsyncModelRunnerOutput):
self.model_runner_output = model_runner_output
self.pooler_output = pooler_output
self.is_valid = is_valid
self.copy_event = torch.cuda.Event()
# Blocking (sleep) event to avoid busy-polling the CUDA driver lock.
self.copy_event = torch.cuda.Event(blocking=True)
with stream(copy_stream, main_stream):
copy_stream.wait_stream(main_stream)
+2 -1
View File
@@ -12,7 +12,8 @@ class DraftTokensHandler:
def __init__(self, device: torch.device | None = None):
self.device = device
self.copy_stream = torch.cuda.Stream(device)
self.copy_event = torch.cuda.Event()
# Blocking (sleep) event to avoid busy-polling the CUDA driver lock.
self.copy_event = torch.cuda.Event(blocking=True)
self.req_ids: list[str] = []
self.draft_tokens_np: np.ndarray | None = None
+7 -3
View File
@@ -259,7 +259,8 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput):
self._invalid_req_indices = invalid_req_indices
# Event on the copy stream so we can synchronize the non-blocking copy.
self.async_copy_ready_event = torch.Event()
# Blocking (sleep) event to avoid busy-polling the CUDA driver lock.
self.async_copy_ready_event = torch.cuda.Event(blocking=True)
# Keep a reference to the device tensor to avoid it being
# deallocated until we finish copying it to the host.
@@ -392,7 +393,8 @@ class AsyncGPUPoolingModelRunnerOutput(AsyncModelRunnerOutput):
self._model_runner_output = model_runner_output
# Event on the copy stream so we can synchronize the non-blocking copy.
self.async_copy_ready_event = torch.Event()
# Blocking (sleep) event to avoid busy-polling the CUDA driver lock.
self.async_copy_ready_event = torch.cuda.Event(blocking=True)
# Keep a reference to the device tensors to avoid them being
# deallocated until we finish copying it to the host.
@@ -718,7 +720,9 @@ class GPUModelRunner(
self.prepare_inputs_event: torch.Event | None = None
if self.use_async_scheduling:
self.async_output_copy_stream = torch.cuda.Stream()
self.prepare_inputs_event = torch.Event()
# Blocking (sleep) event to avoid busy-polling the CUDA driver lock;
# under TP contention that spin can balloon and make the rank a straggler.
self.prepare_inputs_event = torch.cuda.Event(blocking=True)
# self.cudagraph_batch_sizes sorts in ascending order.
if (