[Platform] Replace torch.cuda.Event with torch.Event (#47140)

Signed-off-by: Kunshang Ji <kunshang.ji@intel.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Kunshang Ji
2026-06-30 08:39:59 -07:00
committed by GitHub
co-authored by Harry Mellor
parent fcaa84efa7
commit e840f0d3f5
31 changed files with 60 additions and 70 deletions
+2 -4
View File
@@ -132,10 +132,8 @@ def benchmark_function(
reset_memory_stats()
# Benchmark
start_events = [
torch.cuda.Event(enable_timing=True) for _ in range(benchmark_iters)
]
end_events = [torch.cuda.Event(enable_timing=True) for _ in range(benchmark_iters)]
start_events = [torch.Event(enable_timing=True) for _ in range(benchmark_iters)]
end_events = [torch.Event(enable_timing=True) for _ in range(benchmark_iters)]
for i in range(benchmark_iters):
logits_copy = logits.clone()
+2 -2
View File
@@ -134,8 +134,8 @@ def benchmark_config(
torch.accelerator.synchronize()
# Benchmark
start = torch.cuda.Event(enable_timing=True)
end = torch.cuda.Event(enable_timing=True)
start = torch.Event(enable_timing=True)
end = torch.Event(enable_timing=True)
start.record()
for _ in range(num_iters):
with override_config(config):
@@ -170,8 +170,8 @@ def benchmark_config(
graph.replay()
torch.accelerator.synchronize()
start = torch.cuda.Event(enable_timing=True)
end = torch.cuda.Event(enable_timing=True)
start = torch.Event(enable_timing=True)
end = torch.Event(enable_timing=True)
latencies: list[float] = []
for _ in range(num_iters):
start.record()
@@ -33,7 +33,7 @@ def hf3fs_stats():
def _make_cuda_event():
"""Return a real CUDA event when available, otherwise a MagicMock."""
if torch.cuda.is_available():
return torch.cuda.Event()
return torch.Event()
return MagicMock()
+9 -1
View File
@@ -9,7 +9,7 @@ import regex as re
# --------------------------------------------------------------------------- #
_TORCH_CUDA_PATTERNS = [
r"\btorch\.cuda\.(empty_cache|synchronize|device_count|current_device|memory_reserved|memory_allocated|max_memory_allocated|max_memory_reserved|reset_peak_memory_stats|memory_stats|mem_get_info|set_device|device\()\b",
r"\btorch\.cuda\.(manual_seed|manual_seed_all)\b",
r"\btorch\.cuda\.(manual_seed|manual_seed_all|Event)\b",
r"\bwith\storch\.cuda\.device\b",
# Calls torch.cuda.{_is_compiled/_device_count_amdsmi/_device_count_nvml} internally
r"\bcuda_device_count_stateless\(\)\b",
@@ -21,6 +21,7 @@ ALLOWED_FILES = {
"vllm/device_allocator/",
"vllm/distributed/weight_transfer/ipc_engine.py",
"tests/distributed/test_packed_tensor.py",
"tools/pre_commit/check_torch_cuda.py",
}
@@ -39,6 +40,13 @@ def scan_file(path: str) -> int:
f"Found {matched_text} API call. Use set_random_seed instead."
)
return 1
if matched_text == "torch.cuda.Event":
print(
f"{path}:{line_num}: "
"\033[91merror:\033[0m "
"Found torch.cuda.Event API call. Use torch.Event instead."
)
return 1
print(
f"{path}:{line_num}: "
"\033[91merror:\033[0m " # red color
+2 -2
View File
@@ -742,8 +742,8 @@ class EplbState:
is_main_rank = ep_rank == 0
if is_main_rank:
if not self.is_async or is_profile:
start_event = torch.cuda.Event(enable_timing=True)
end_event = torch.cuda.Event(enable_timing=True)
start_event = torch.Event(enable_timing=True)
end_event = torch.Event(enable_timing=True)
start_event.record()
logger.info(
"Rearranging experts %s %s...",
+2 -2
View File
@@ -31,7 +31,7 @@ class CpuGpuEvent:
"""
def __init__(self):
self._event = torch.cuda.Event()
self._event = torch.Event()
self._recorded = threading.Event()
def wait(self, stream: torch.cuda.Stream | None = None):
@@ -56,7 +56,7 @@ class CpuGpuEvent:
"CpuGpuEvent.record() called before the previous event was "
"consumed by wait()"
)
self._event = torch.cuda.Event()
self._event = torch.Event()
self._event.record(stream)
self._recorded.set()
@@ -239,7 +239,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
# this event is complete the request is considered "done sending"
# by get_finished; clients block on the per-file flock to wait for
# the disk write itself.
self._req_copy_events: dict[str, torch.cuda.Event] = {}
self._req_copy_events: dict[str, torch.Event] = {}
# req_ids reported as finished-generating by the scheduler,
# accumulated across get_finished calls.
self._accumulated_finished_req_ids: set[str] = set()
@@ -320,7 +320,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
@staticmethod
def _write_tensors(
tensors: dict[str, torch.Tensor],
event: torch.cuda.Event,
event: torch.Event,
filename: str,
lock_fd: int | None,
) -> None:
@@ -375,7 +375,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
copy_stream = self._get_copy_stream()
# Ensure the copy stream sees all prior writes on the default stream.
ready_event = torch.cuda.Event()
ready_event = torch.Event()
ready_event.record()
copy_stream.wait_event(ready_event)
@@ -396,7 +396,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
pinned_hs.copy_(hidden_states_gpu, non_blocking=True)
# Record completion of this copy on the copy stream.
copy_done = torch.cuda.Event()
copy_done = torch.Event()
copy_done.record(copy_stream)
# token_ids is already on CPU (created in request_finished).
@@ -221,7 +221,7 @@ class Hf3fsClient:
@wsynchronized()
def batch_write(
self, offsets: list[int], tensors: list[torch.Tensor], event: torch.cuda.Event
self, offsets: list[int], tensors: list[torch.Tensor], event: torch.Event
) -> list[int]:
"""Write data from tensors to the file at specified offsets.
@@ -133,7 +133,7 @@ class AsyncOperationManager:
# CUDA streams for async operations
self._save_stream = torch.cuda.Stream()
self._load_stream = torch.cuda.Stream()
self._save_event = torch.cuda.Event()
self._save_event = torch.Event()
# Buffer allocators for data copying
self._save_buffer_allocator = CopyBufferAllocator(
@@ -171,7 +171,7 @@ class AsyncOperationManager:
def submit_save_operation(self, request_id: str, block_ids, block_hashes) -> Future:
"""Submit a save operation for async execution."""
future: Future[Any] = Future()
main_stream_event = torch.cuda.Event()
main_stream_event = torch.Event()
main_stream_event.record()
task = (request_id, block_ids, block_hashes, future, main_stream_event)
self._save_queue.put(task)
@@ -304,7 +304,7 @@ class AsyncOperationManager:
block_ids, buffers, "gather"
)
save_stream_event = torch.cuda.Event()
save_stream_event = torch.Event()
save_stream_event.record(self._save_stream) # Record gather completion
# Step3: Write data in batches
@@ -75,7 +75,7 @@ class Hf3fsClient:
return torch.frombuffer(buffer_data, dtype=dtype)
def batch_write(
self, offsets: list[int], tensors: list[torch.Tensor], event: torch.cuda.Event
self, offsets: list[int], tensors: list[torch.Tensor], event: torch.Event
) -> list[int]:
"""Write data from tensors to file at specified offsets."""
results = []
@@ -430,7 +430,7 @@ class LMCacheMPWorkerAdapter:
@_lmcache_nvtx_annotate
def submit_store_request(
self, request_id: str, op: LoadStoreOp, event: torch.cuda.Event
self, request_id: str, op: LoadStoreOp, event: torch.Event
):
"""
Submit a KV cache store request to LMCache
@@ -464,7 +464,7 @@ class LMCacheMPWorkerAdapter:
@_lmcache_nvtx_annotate
def submit_retrieve_request(
self, request_id: str, op: LoadStoreOp, event: torch.cuda.Event
self, request_id: str, op: LoadStoreOp, event: torch.Event
):
"""
Submit a KV cache retrieve request to LMCache
@@ -501,7 +501,7 @@ class LMCacheMPWorkerAdapter:
self,
request_ids: list[str],
ops: list[LoadStoreOp],
event: torch.cuda.Event,
event: torch.Event,
):
"""
Submit a batched store request to LMCache
@@ -550,7 +550,7 @@ class LMCacheMPWorkerAdapter:
self,
request_ids: list[str],
ops: list[LoadStoreOp],
event: torch.cuda.Event,
event: torch.Event,
):
"""
Submit a batched retrieve request to LMCache
@@ -589,7 +589,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
return
with torch.cuda.stream(torch.cuda.current_stream()):
event = torch.cuda.Event(interprocess=True)
event = torch.Event(interprocess=True)
event.record()
self.worker_adapter.batched_submit_retrieve_requests(
@@ -663,7 +663,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
return
with torch.cuda.stream(torch.cuda.current_stream()):
event = torch.cuda.Event(interprocess=True)
event = torch.Event(interprocess=True)
event.record()
self.worker_adapter.batched_submit_store_requests(
@@ -323,7 +323,7 @@ class ReqMeta:
can_save: bool | None = None
load_spec: LoadSpec | None = None
is_last_chunk: bool | None = None
current_event: torch.cuda.Event | None = None
current_event: torch.Event | None = None
token_ids: list[int] | None = None
num_prompt_tokens: int | None = None
@@ -1357,7 +1357,7 @@ class MooncakeStoreWorker:
current_event = None
for request in meta.requests:
if request.can_save:
current_event = torch.cuda.Event()
current_event = torch.Event()
current_event.record()
break
@@ -56,7 +56,7 @@ class WriteTask:
local_block_ids: list[int]
remote_block_ids_hint: list[int] | None
layer_name: str
event: torch.cuda.Event
event: torch.Event
remote_notify_port: int
remote_ip: str
enqueue_time: float = field(default_factory=time.perf_counter)
@@ -1061,7 +1061,7 @@ class MoRIIOConnectorWorker:
# when mori-io supports ibgda functionality
stream = torch.cuda.current_stream()
event = torch.cuda.Event()
event = torch.Event()
event.record(stream)
task = WriteTask(
+1 -1
View File
@@ -89,7 +89,7 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
vllm_config = get_current_vllm_config()
self._lora_stream = _get_lora_aux_cuda_stream()
assert current_platform.is_cuda_alike()
self._events = [torch.cuda.Event(), torch.cuda.Event()]
self._events = [torch.Event(), torch.Event()]
# lora_linear avoids prefix conflicts with the base layer
self.layer_name = self.base_layer.prefix + ".lora_linear_async"
compilation_config = vllm_config.compilation_config
+2 -2
View File
@@ -118,7 +118,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
def _init_lora_stream_context(self) -> None:
self._lora_stream: torch.cuda.Stream | None = None
self._events: tuple[torch.cuda.Event, ...] | None = None
self._events: tuple[torch.Event, ...] | None = None
if not self._enable_aux_cuda_stream:
return
if not current_platform.is_cuda_alike():
@@ -127,7 +127,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
# 4 events: 2 per (base GEMM, LoRA) pair so w13 and w2 don't reuse
# the same event objects; reuse-within-a-pair is fine because the
# second pair starts only after intermediate_cache1.add_() has joined.
self._events = tuple(torch.cuda.Event() for _ in range(4))
self._events = tuple(torch.Event() for _ in range(4))
def _build_lora_context(self):
use_dual_stream = (
@@ -51,7 +51,7 @@ class MoELoRAContext:
# Events are paired one-per-overlap-pair: events[0,1] for w13,
# events[2,3] for w2, so the two pairs do not race on the same event.
aux_stream: torch.cuda.Stream | None = None
events: tuple[torch.cuda.Event, ...] | None = None
events: tuple[torch.Event, ...] | None = None
# Per-rank token→LoRA mapping after EP dispatch. Set by
# FusedMoEPrepareAndFinalizeModular.prepare() when EP+LoRA is active, read
+2 -2
View File
@@ -388,7 +388,7 @@ class _ModuleOffloader:
# Event to signal when H2D copy to static buffer is complete.
# Used for per-layer synchronization (both eager and capture modes).
self._copy_done_event = torch.cuda.Event()
self._copy_done_event = torch.Event()
# Track whether _copy_done_event is valid for eager-mode wait_event.
# False when: (1) never recorded, or (2) last recorded during a
@@ -518,7 +518,7 @@ class _ModuleOffloader:
# Fork: record event on compute stream, copy_stream waits on it
# This joins copy_stream to any active CUDA graph capture
fork_event = torch.cuda.Event()
fork_event = torch.Event()
torch.cuda.current_stream().record_event(fork_event)
self.copy_stream.wait_event(fork_event)
+2 -5
View File
@@ -272,7 +272,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
# [0]: GEMM start / post-GEMM event0. [1..3]: GEMM done events;
# [1] doubles as post-GEMM event1. Reuse is safe: GEMM fully joins
# before post-GEMM starts.
self.ln_events = [torch.cuda.Event() for _ in range(4)]
self.ln_events = [torch.Event() for _ in range(4)]
assert cache_config is not None, "DeepseekV4 attention requires cache_config"
# ---- Attention / KV-cache setup ----
@@ -760,10 +760,7 @@ class DeepseekV4Indexer(nn.Module):
# None on ROCm — maybe_execute_in_parallel falls back to sequential.
self.aux_stream = aux_stream
self.ln_events: list[torch.cuda.Event] = [
torch.cuda.Event(),
torch.cuda.Event(),
]
self.ln_events: list[torch.Event] = [torch.Event(), torch.Event()]
def forward(
self,
-11
View File
@@ -44,17 +44,6 @@ class DeepseekV4XPUAttention(DeepseekV4Attention):
backend_cls = DeepseekV4XPUSparseBackend
use_flashmla_fp8_layout = True
def __init__(self, *args, **kwargs) -> None:
# torch.cuda.Event() raises RuntimeError on XPU ("dummy base class").
# The Base and DeepseekV4Indexer both create cuda Events in __init__, so
# we temporarily redirect torch.cuda.Event → torch.xpu.Event.
_orig_event = torch.cuda.Event
torch.cuda.Event = torch.xpu.Event # type: ignore[misc]
try:
super().__init__(*args, **kwargs)
finally:
torch.cuda.Event = _orig_event # type: ignore[misc]
def _fused_qnorm_rope_kv_insert(self, q, kv, positions, attn_metadata):
from typing import cast
+5 -5
View File
@@ -20,8 +20,8 @@ class EventType(Enum):
def maybe_execute_in_parallel(
fn0: Callable[[], Any],
fn1: Callable[[], Any],
event0: torch.cuda.Event,
event1: torch.cuda.Event,
event0: torch.Event,
event1: torch.Event,
aux_stream: torch.cuda.Stream | None = None,
) -> tuple[Any, Any]:
"""Run two functions potentially in parallel on separate CUDA streams.
@@ -61,8 +61,8 @@ def maybe_execute_in_parallel(
def execute_in_parallel(
default_fn: Callable[[], Any],
aux_fns: list[Callable[[], Any] | None],
start_event: torch.cuda.Event,
done_events: list[torch.cuda.Event],
start_event: torch.Event,
done_events: list[torch.Event],
aux_streams: list[torch.cuda.Stream] | None = None,
enable: bool = False,
) -> tuple[Any, list[Any]]:
@@ -108,7 +108,7 @@ def execute_in_parallel(
)
aux_results = [None] * len(aux_fns)
pending: list[torch.cuda.Event] = []
pending: list[torch.Event] = []
start_event.record()
for i, fn in enumerate(aux_fns):
+2 -2
View File
@@ -468,7 +468,7 @@ class NgramProposerGPU:
def update_scheduler_for_invalid_drafts(
num_valid_draft_tokens_event: torch.cuda.Event,
num_valid_draft_tokens_event: torch.Event,
num_valid_draft_tokens_cpu: torch.Tensor,
scheduler_output: "SchedulerOutput",
req_id_to_index: dict[str, int],
@@ -643,7 +643,7 @@ def _sync_num_tokens(
def copy_num_valid_draft_tokens(
num_valid_draft_tokens_cpu: torch.Tensor,
num_valid_draft_tokens_copy_stream: torch.cuda.Stream,
num_valid_draft_tokens_event: torch.cuda.Event,
num_valid_draft_tokens_event: torch.Event,
num_valid_draft_tokens: torch.Tensor | None,
batch_size: int,
) -> None:
-1
View File
@@ -47,7 +47,6 @@ def get_memory_info(*args: Any, **kwargs: Any) -> tuple[int, int]:
torch.Event = _EventPlaceholder
torch.cuda.Event = _EventPlaceholder
torch.cuda.Stream = _StreamPlaceholder
torch.cuda.set_stream = noop
torch.cuda.current_stream = lambda *args, **kwargs: _StreamPlaceholder()
+2 -2
View File
@@ -24,7 +24,7 @@ class AsyncOutput(AsyncModelRunnerOutput):
self.model_runner_output = model_runner_output
self.sampler_output = sampler_output
self.num_sampled_tokens = num_sampled_tokens
self.copy_event = torch.cuda.Event()
self.copy_event = torch.Event()
with stream(copy_stream, main_stream):
copy_stream.wait_stream(main_stream)
@@ -81,7 +81,7 @@ class AsyncPoolingOutput(AsyncModelRunnerOutput):
self.model_runner_output = model_runner_output
self.pooler_output = pooler_output
self.is_valid = is_valid
self.copy_event = torch.cuda.Event()
self.copy_event = torch.Event()
with stream(copy_stream, main_stream):
copy_stream.wait_stream(main_stream)
+1 -1
View File
@@ -17,7 +17,7 @@ from vllm.v1.worker.gpu.input_batch import InputBatch
class PendingRecv:
"""Per-step slot data for a deferred postprocess on the main stream."""
event: torch.cuda.Event
event: torch.Event
sampled_tokens: torch.Tensor # [num_reqs, max_sample_len]
num_sampled: torch.Tensor # [num_reqs]
+1 -1
View File
@@ -12,7 +12,7 @@ class DraftTokensHandler:
def __init__(self, device: torch.device | None = None):
self.device = device
self.copy_stream = torch.cuda.Stream(device)
self.copy_event = torch.cuda.Event()
self.copy_event = torch.Event()
self.req_ids: list[str] = []
self.draft_tokens_np: np.ndarray | None = None
+2 -2
View File
@@ -839,7 +839,7 @@ class GPUModelRunner(
# N-gram GPU path: async D2H buffer/event for per-request valid draft counts.
self._num_valid_draft_tokens: torch.Tensor | None = None
self._num_valid_draft_tokens_cpu: torch.Tensor | None = None
self._num_valid_draft_tokens_event: torch.cuda.Event | None = None
self._num_valid_draft_tokens_event: torch.Event | None = None
self._num_valid_draft_tokens_copy_stream: torch.cuda.Stream | None = None
if (
self.speculative_config is not None
@@ -848,7 +848,7 @@ class GPUModelRunner(
self._num_valid_draft_tokens_cpu = torch.empty(
self.max_num_reqs, dtype=torch.int32, pin_memory=PIN_MEMORY
)
self._num_valid_draft_tokens_event = torch.cuda.Event()
self._num_valid_draft_tokens_event = torch.Event()
self._num_valid_draft_tokens_copy_stream = torch.cuda.Stream()
self._draft_token_req_ids: list[str] | None = None
-1
View File
@@ -45,7 +45,6 @@ def _torch_cuda_wrapper():
torch.cuda.default_stream = torch.xpu.current_stream
torch.cuda.current_stream = torch.xpu.current_stream
torch.cuda.stream = torch.xpu.stream
torch.cuda.Event = torch.Event
torch.cuda.set_stream = torch.xpu.set_stream
if supports_xpu_graph():
torch.cuda.graph = torch.xpu.graph