[Perf] Use blocking CUDA events to avoid busy polling cuda driver lock (#47081)

Signed-off-by: Jingyi Yang <girasoleyang@gmail.com>
This commit is contained in:
Summer Yang
2026-07-07 09:36:10 +00:00
committed by GitHub
parent c85d72076a
commit d3e69fd671
3 changed files with 13 additions and 6 deletions
+2 -1
View File
@@ -12,7 +12,8 @@ class DraftTokensHandler:
def __init__(self, device: torch.device | None = None):
self.device = device
self.copy_stream = torch.cuda.Stream(device)
self.copy_event = torch.cuda.Event()
# Blocking (sleep) event to avoid busy-polling the CUDA driver lock.
self.copy_event = torch.cuda.Event(blocking=True)
self.req_ids: list[str] = []
self.draft_tokens_np: np.ndarray | None = None