diff --git a/benchmarks/benchmark_topk_topp.py b/benchmarks/benchmark_topk_topp.py index 27b6dd8d6b..f727f16ea2 100644 --- a/benchmarks/benchmark_topk_topp.py +++ b/benchmarks/benchmark_topk_topp.py @@ -132,8 +132,10 @@ def benchmark_function( reset_memory_stats() # Benchmark - start_events = [torch.Event(enable_timing=True) for _ in range(benchmark_iters)] - end_events = [torch.Event(enable_timing=True) for _ in range(benchmark_iters)] + start_events = [ + torch.cuda.Event(enable_timing=True) for _ in range(benchmark_iters) + ] + end_events = [torch.cuda.Event(enable_timing=True) for _ in range(benchmark_iters)] for i in range(benchmark_iters): logits_copy = logits.clone() diff --git a/benchmarks/kernels/benchmark_moe_defaults.py b/benchmarks/kernels/benchmark_moe_defaults.py index 7f000e0113..f6ad59366d 100644 --- a/benchmarks/kernels/benchmark_moe_defaults.py +++ b/benchmarks/kernels/benchmark_moe_defaults.py @@ -134,8 +134,8 @@ def benchmark_config( torch.accelerator.synchronize() # Benchmark - start = torch.Event(enable_timing=True) - end = torch.Event(enable_timing=True) + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) start.record() for _ in range(num_iters): with override_config(config): diff --git a/benchmarks/kernels/benchmark_selective_state_update.py b/benchmarks/kernels/benchmark_selective_state_update.py index 5a3a6e88a6..a8b73da2aa 100644 --- a/benchmarks/kernels/benchmark_selective_state_update.py +++ b/benchmarks/kernels/benchmark_selective_state_update.py @@ -170,8 +170,8 @@ def benchmark_config( graph.replay() torch.accelerator.synchronize() - start = torch.Event(enable_timing=True) - end = torch.Event(enable_timing=True) + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) latencies: list[float] = [] for _ in range(num_iters): start.record() diff --git a/tests/v1/kv_connector/unit/test_hf3fs_connector.py b/tests/v1/kv_connector/unit/test_hf3fs_connector.py index cd525e23b1..94bb94c6fb 100644 --- a/tests/v1/kv_connector/unit/test_hf3fs_connector.py +++ b/tests/v1/kv_connector/unit/test_hf3fs_connector.py @@ -33,7 +33,7 @@ def hf3fs_stats(): def _make_cuda_event(): """Return a real CUDA event when available, otherwise a MagicMock.""" if torch.cuda.is_available(): - return torch.Event() + return torch.cuda.Event() return MagicMock() diff --git a/tools/pre_commit/check_torch_cuda.py b/tools/pre_commit/check_torch_cuda.py index 9a67a013f1..aec7b85d59 100644 --- a/tools/pre_commit/check_torch_cuda.py +++ b/tools/pre_commit/check_torch_cuda.py @@ -9,7 +9,7 @@ import regex as re # --------------------------------------------------------------------------- # _TORCH_CUDA_PATTERNS = [ r"\btorch\.cuda\.(empty_cache|synchronize|device_count|current_device|memory_reserved|memory_allocated|max_memory_allocated|max_memory_reserved|reset_peak_memory_stats|memory_stats|mem_get_info|set_device|device\()\b", - r"\btorch\.cuda\.(manual_seed|manual_seed_all|Event)\b", + r"\btorch\.cuda\.(manual_seed|manual_seed_all)\b", r"\bwith\storch\.cuda\.device\b", # Calls torch.cuda.{_is_compiled/_device_count_amdsmi/_device_count_nvml} internally r"\bcuda_device_count_stateless\(\)\b", @@ -21,7 +21,6 @@ ALLOWED_FILES = { "vllm/device_allocator/", "vllm/distributed/weight_transfer/ipc_engine.py", "tests/distributed/test_packed_tensor.py", - "tools/pre_commit/check_torch_cuda.py", } @@ -40,13 +39,6 @@ def scan_file(path: str) -> int: f"Found {matched_text} API call. Use set_random_seed instead." ) return 1 - if matched_text == "torch.cuda.Event": - print( - f"{path}:{line_num}: " - "\033[91merror:\033[0m " - "Found torch.cuda.Event API call. Use torch.Event instead." - ) - return 1 print( f"{path}:{line_num}: " "\033[91merror:\033[0m " # red color diff --git a/vllm/distributed/eplb/eplb_state.py b/vllm/distributed/eplb/eplb_state.py index e98c765f53..feacb03d28 100644 --- a/vllm/distributed/eplb/eplb_state.py +++ b/vllm/distributed/eplb/eplb_state.py @@ -742,8 +742,8 @@ class EplbState: is_main_rank = ep_rank == 0 if is_main_rank: if not self.is_async or is_profile: - start_event = torch.Event(enable_timing=True) - end_event = torch.Event(enable_timing=True) + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) start_event.record() logger.info( "Rearranging experts %s %s...", diff --git a/vllm/distributed/eplb/eplb_utils.py b/vllm/distributed/eplb/eplb_utils.py index 21a7ee68fa..dee1974974 100644 --- a/vllm/distributed/eplb/eplb_utils.py +++ b/vllm/distributed/eplb/eplb_utils.py @@ -31,7 +31,7 @@ class CpuGpuEvent: """ def __init__(self): - self._event = torch.Event() + self._event = torch.cuda.Event() self._recorded = threading.Event() def wait(self, stream: torch.cuda.Stream | None = None): @@ -56,7 +56,7 @@ class CpuGpuEvent: "CpuGpuEvent.record() called before the previous event was " "consumed by wait()" ) - self._event = torch.Event() + self._event = torch.cuda.Event() self._event.record(stream) self._recorded.set() diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py index a604bd5528..299ff037ad 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py @@ -239,7 +239,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA): # this event is complete the request is considered "done sending" # by get_finished; clients block on the per-file flock to wait for # the disk write itself. - self._req_copy_events: dict[str, torch.Event] = {} + self._req_copy_events: dict[str, torch.cuda.Event] = {} # req_ids reported as finished-generating by the scheduler, # accumulated across get_finished calls. self._accumulated_finished_req_ids: set[str] = set() @@ -320,7 +320,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA): @staticmethod def _write_tensors( tensors: dict[str, torch.Tensor], - event: torch.Event, + event: torch.cuda.Event, filename: str, lock_fd: int | None, ) -> None: @@ -375,7 +375,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA): copy_stream = self._get_copy_stream() # Ensure the copy stream sees all prior writes on the default stream. - ready_event = torch.Event() + ready_event = torch.cuda.Event() ready_event.record() copy_stream.wait_event(ready_event) @@ -396,7 +396,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA): pinned_hs.copy_(hidden_states_gpu, non_blocking=True) # Record completion of this copy on the copy stream. - copy_done = torch.Event() + copy_done = torch.cuda.Event() copy_done.record(copy_stream) # token_ids is already on CPU (created in request_finished). diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/hf3fs_client.py b/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/hf3fs_client.py index edcf83b192..a54233453b 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/hf3fs_client.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/hf3fs_client.py @@ -221,7 +221,7 @@ class Hf3fsClient: @wsynchronized() def batch_write( - self, offsets: list[int], tensors: list[torch.Tensor], event: torch.Event + self, offsets: list[int], tensors: list[torch.Tensor], event: torch.cuda.Event ) -> list[int]: """Write data from tensors to the file at specified offsets. diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/hf3fs_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/hf3fs_connector.py index 55a8b5a161..526375952f 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/hf3fs_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/hf3fs_connector.py @@ -133,7 +133,7 @@ class AsyncOperationManager: # CUDA streams for async operations self._save_stream = torch.cuda.Stream() self._load_stream = torch.cuda.Stream() - self._save_event = torch.Event() + self._save_event = torch.cuda.Event() # Buffer allocators for data copying self._save_buffer_allocator = CopyBufferAllocator( @@ -171,7 +171,7 @@ class AsyncOperationManager: def submit_save_operation(self, request_id: str, block_ids, block_hashes) -> Future: """Submit a save operation for async execution.""" future: Future[Any] = Future() - main_stream_event = torch.Event() + main_stream_event = torch.cuda.Event() main_stream_event.record() task = (request_id, block_ids, block_hashes, future, main_stream_event) self._save_queue.put(task) @@ -304,7 +304,7 @@ class AsyncOperationManager: block_ids, buffers, "gather" ) - save_stream_event = torch.Event() + save_stream_event = torch.cuda.Event() save_stream_event.record(self._save_stream) # Record gather completion # Step3: Write data in batches diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/utils/hf3fs_mock_client.py b/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/utils/hf3fs_mock_client.py index e2718d1faa..3914663a62 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/utils/hf3fs_mock_client.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/utils/hf3fs_mock_client.py @@ -75,7 +75,7 @@ class Hf3fsClient: return torch.frombuffer(buffer_data, dtype=dtype) def batch_write( - self, offsets: list[int], tensors: list[torch.Tensor], event: torch.Event + self, offsets: list[int], tensors: list[torch.Tensor], event: torch.cuda.Event ) -> list[int]: """Write data from tensors to file at specified offsets.""" results = [] diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_integration/multi_process_adapter.py b/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_integration/multi_process_adapter.py index 6d83380cad..2e75519df1 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_integration/multi_process_adapter.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_integration/multi_process_adapter.py @@ -430,7 +430,7 @@ class LMCacheMPWorkerAdapter: @_lmcache_nvtx_annotate def submit_store_request( - self, request_id: str, op: LoadStoreOp, event: torch.Event + self, request_id: str, op: LoadStoreOp, event: torch.cuda.Event ): """ Submit a KV cache store request to LMCache @@ -464,7 +464,7 @@ class LMCacheMPWorkerAdapter: @_lmcache_nvtx_annotate def submit_retrieve_request( - self, request_id: str, op: LoadStoreOp, event: torch.Event + self, request_id: str, op: LoadStoreOp, event: torch.cuda.Event ): """ Submit a KV cache retrieve request to LMCache @@ -501,7 +501,7 @@ class LMCacheMPWorkerAdapter: self, request_ids: list[str], ops: list[LoadStoreOp], - event: torch.Event, + event: torch.cuda.Event, ): """ Submit a batched store request to LMCache @@ -550,7 +550,7 @@ class LMCacheMPWorkerAdapter: self, request_ids: list[str], ops: list[LoadStoreOp], - event: torch.Event, + event: torch.cuda.Event, ): """ Submit a batched retrieve request to LMCache diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_mp_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_mp_connector.py index 2ca35be2b5..8786e91a5a 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_mp_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_mp_connector.py @@ -589,7 +589,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1): return with torch.cuda.stream(torch.cuda.current_stream()): - event = torch.Event(interprocess=True) + event = torch.cuda.Event(interprocess=True) event.record() self.worker_adapter.batched_submit_retrieve_requests( @@ -663,7 +663,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1): return with torch.cuda.stream(torch.cuda.current_stream()): - event = torch.Event(interprocess=True) + event = torch.cuda.Event(interprocess=True) event.record() self.worker_adapter.batched_submit_store_requests( diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py index 3b69ce9a17..ef98ec0d4e 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py @@ -323,7 +323,7 @@ class ReqMeta: can_save: bool | None = None load_spec: LoadSpec | None = None is_last_chunk: bool | None = None - current_event: torch.Event | None = None + current_event: torch.cuda.Event | None = None token_ids: list[int] | None = None num_prompt_tokens: int | None = None diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py index 818c2479a1..e60d2f47a4 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py @@ -1357,7 +1357,7 @@ class MooncakeStoreWorker: current_event = None for request in meta.requests: if request.can_save: - current_event = torch.Event() + current_event = torch.cuda.Event() current_event.record() break diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py index 7b8ab56605..15585123e5 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py @@ -56,7 +56,7 @@ class WriteTask: local_block_ids: list[int] remote_block_ids_hint: list[int] | None layer_name: str - event: torch.Event + event: torch.cuda.Event remote_notify_port: int remote_ip: str enqueue_time: float = field(default_factory=time.perf_counter) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py index 172836f5cc..de21a1398e 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py @@ -1061,7 +1061,7 @@ class MoRIIOConnectorWorker: # when mori-io supports ibgda functionality stream = torch.cuda.current_stream() - event = torch.Event() + event = torch.cuda.Event() event.record(stream) task = WriteTask( diff --git a/vllm/lora/layers/base_linear.py b/vllm/lora/layers/base_linear.py index bff3c0cf45..5c8e829b29 100644 --- a/vllm/lora/layers/base_linear.py +++ b/vllm/lora/layers/base_linear.py @@ -89,7 +89,7 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA): vllm_config = get_current_vllm_config() self._lora_stream = _get_lora_aux_cuda_stream() assert current_platform.is_cuda_alike() - self._events = [torch.Event(), torch.Event()] + self._events = [torch.cuda.Event(), torch.cuda.Event()] # lora_linear avoids prefix conflicts with the base layer self.layer_name = self.base_layer.prefix + ".lora_linear_async" compilation_config = vllm_config.compilation_config diff --git a/vllm/lora/layers/fused_moe.py b/vllm/lora/layers/fused_moe.py index 39f60aad5d..63a4ea9a82 100644 --- a/vllm/lora/layers/fused_moe.py +++ b/vllm/lora/layers/fused_moe.py @@ -118,7 +118,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA): def _init_lora_stream_context(self) -> None: self._lora_stream: torch.cuda.Stream | None = None - self._events: tuple[torch.Event, ...] | None = None + self._events: tuple[torch.cuda.Event, ...] | None = None if not self._enable_aux_cuda_stream: return if not current_platform.is_cuda_alike(): @@ -127,7 +127,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA): # 4 events: 2 per (base GEMM, LoRA) pair so w13 and w2 don't reuse # the same event objects; reuse-within-a-pair is fine because the # second pair starts only after intermediate_cache1.add_() has joined. - self._events = tuple(torch.Event() for _ in range(4)) + self._events = tuple(torch.cuda.Event() for _ in range(4)) def _build_lora_context(self): use_dual_stream = ( diff --git a/vllm/model_executor/layers/fused_moe/experts/lora_context.py b/vllm/model_executor/layers/fused_moe/experts/lora_context.py index dd7429d76e..117f744aee 100644 --- a/vllm/model_executor/layers/fused_moe/experts/lora_context.py +++ b/vllm/model_executor/layers/fused_moe/experts/lora_context.py @@ -51,7 +51,7 @@ class MoELoRAContext: # Events are paired one-per-overlap-pair: events[0,1] for w13, # events[2,3] for w2, so the two pairs do not race on the same event. aux_stream: torch.cuda.Stream | None = None - events: tuple[torch.Event, ...] | None = None + events: tuple[torch.cuda.Event, ...] | None = None # Per-rank token→LoRA mapping after EP dispatch. Set by # FusedMoEPrepareAndFinalizeModular.prepare() when EP+LoRA is active, read diff --git a/vllm/model_executor/offloader/prefetch.py b/vllm/model_executor/offloader/prefetch.py index 2c86063265..466d8c13ce 100644 --- a/vllm/model_executor/offloader/prefetch.py +++ b/vllm/model_executor/offloader/prefetch.py @@ -388,7 +388,7 @@ class _ModuleOffloader: # Event to signal when H2D copy to static buffer is complete. # Used for per-layer synchronization (both eager and capture modes). - self._copy_done_event = torch.Event() + self._copy_done_event = torch.cuda.Event() # Track whether _copy_done_event is valid for eager-mode wait_event. # False when: (1) never recorded, or (2) last recorded during a @@ -518,7 +518,7 @@ class _ModuleOffloader: # Fork: record event on compute stream, copy_stream waits on it # This joins copy_stream to any active CUDA graph capture - fork_event = torch.Event() + fork_event = torch.cuda.Event() torch.cuda.current_stream().record_event(fork_event) self.copy_stream.wait_event(fork_event) diff --git a/vllm/models/deepseek_v4/attention.py b/vllm/models/deepseek_v4/attention.py index 519f5f9a14..29a19d9026 100644 --- a/vllm/models/deepseek_v4/attention.py +++ b/vllm/models/deepseek_v4/attention.py @@ -272,7 +272,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): # [0]: GEMM start / post-GEMM event0. [1..3]: GEMM done events; # [1] doubles as post-GEMM event1. Reuse is safe: GEMM fully joins # before post-GEMM starts. - self.ln_events = [torch.Event() for _ in range(4)] + self.ln_events = [torch.cuda.Event() for _ in range(4)] assert cache_config is not None, "DeepseekV4 attention requires cache_config" # ---- Attention / KV-cache setup ---- @@ -760,7 +760,10 @@ class DeepseekV4Indexer(nn.Module): # None on ROCm — maybe_execute_in_parallel falls back to sequential. self.aux_stream = aux_stream - self.ln_events: list[torch.Event] = [torch.Event(), torch.Event()] + self.ln_events: list[torch.cuda.Event] = [ + torch.cuda.Event(), + torch.cuda.Event(), + ] def forward( self, diff --git a/vllm/models/deepseek_v4/xpu/xpu_sparse.py b/vllm/models/deepseek_v4/xpu/xpu_sparse.py index 74d27d7bc4..77cc35cf49 100644 --- a/vllm/models/deepseek_v4/xpu/xpu_sparse.py +++ b/vllm/models/deepseek_v4/xpu/xpu_sparse.py @@ -44,6 +44,17 @@ class DeepseekV4XPUAttention(DeepseekV4Attention): backend_cls = DeepseekV4XPUSparseBackend use_flashmla_fp8_layout = True + def __init__(self, *args, **kwargs) -> None: + # torch.cuda.Event() raises RuntimeError on XPU ("dummy base class"). + # The Base and DeepseekV4Indexer both create cuda Events in __init__, so + # we temporarily redirect torch.cuda.Event → torch.xpu.Event. + _orig_event = torch.cuda.Event + torch.cuda.Event = torch.xpu.Event # type: ignore[misc] + try: + super().__init__(*args, **kwargs) + finally: + torch.cuda.Event = _orig_event # type: ignore[misc] + def _fused_qnorm_rope_kv_insert(self, q, kv, positions, attn_metadata): from typing import cast diff --git a/vllm/utils/multi_stream_utils.py b/vllm/utils/multi_stream_utils.py index fed38ea1a3..2203221c5a 100644 --- a/vllm/utils/multi_stream_utils.py +++ b/vllm/utils/multi_stream_utils.py @@ -20,8 +20,8 @@ class EventType(Enum): def maybe_execute_in_parallel( fn0: Callable[[], Any], fn1: Callable[[], Any], - event0: torch.Event, - event1: torch.Event, + event0: torch.cuda.Event, + event1: torch.cuda.Event, aux_stream: torch.cuda.Stream | None = None, ) -> tuple[Any, Any]: """Run two functions potentially in parallel on separate CUDA streams. @@ -61,8 +61,8 @@ def maybe_execute_in_parallel( def execute_in_parallel( default_fn: Callable[[], Any], aux_fns: list[Callable[[], Any] | None], - start_event: torch.Event, - done_events: list[torch.Event], + start_event: torch.cuda.Event, + done_events: list[torch.cuda.Event], aux_streams: list[torch.cuda.Stream] | None = None, enable: bool = False, ) -> tuple[Any, list[Any]]: @@ -108,7 +108,7 @@ def execute_in_parallel( ) aux_results = [None] * len(aux_fns) - pending: list[torch.Event] = [] + pending: list[torch.cuda.Event] = [] start_event.record() for i, fn in enumerate(aux_fns): diff --git a/vllm/v1/spec_decode/ngram_proposer_gpu.py b/vllm/v1/spec_decode/ngram_proposer_gpu.py index a08b341e80..ed544bb27c 100644 --- a/vllm/v1/spec_decode/ngram_proposer_gpu.py +++ b/vllm/v1/spec_decode/ngram_proposer_gpu.py @@ -468,7 +468,7 @@ class NgramProposerGPU: def update_scheduler_for_invalid_drafts( - num_valid_draft_tokens_event: torch.Event, + num_valid_draft_tokens_event: torch.cuda.Event, num_valid_draft_tokens_cpu: torch.Tensor, scheduler_output: "SchedulerOutput", req_id_to_index: dict[str, int], @@ -643,7 +643,7 @@ def _sync_num_tokens( def copy_num_valid_draft_tokens( num_valid_draft_tokens_cpu: torch.Tensor, num_valid_draft_tokens_copy_stream: torch.cuda.Stream, - num_valid_draft_tokens_event: torch.Event, + num_valid_draft_tokens_event: torch.cuda.Event, num_valid_draft_tokens: torch.Tensor | None, batch_size: int, ) -> None: diff --git a/vllm/v1/worker/cpu/shm.py b/vllm/v1/worker/cpu/shm.py index deec52e44b..bd1f96c71e 100644 --- a/vllm/v1/worker/cpu/shm.py +++ b/vllm/v1/worker/cpu/shm.py @@ -48,6 +48,7 @@ def get_memory_info(*args: Any, **kwargs: Any) -> tuple[int, int]: torch.Event = _EventPlaceholder +torch.cuda.Event = _EventPlaceholder torch.cuda.Stream = _StreamPlaceholder torch.cuda.set_stream = noop torch.cuda.current_stream = lambda *args, **kwargs: _StreamPlaceholder() diff --git a/vllm/v1/worker/gpu/async_utils.py b/vllm/v1/worker/gpu/async_utils.py index a9ad16b152..b3d6f5e4d9 100644 --- a/vllm/v1/worker/gpu/async_utils.py +++ b/vllm/v1/worker/gpu/async_utils.py @@ -24,7 +24,7 @@ class AsyncOutput(AsyncModelRunnerOutput): self.model_runner_output = model_runner_output self.sampler_output = sampler_output self.num_sampled_tokens = num_sampled_tokens - self.copy_event = torch.Event() + self.copy_event = torch.cuda.Event() with stream(copy_stream, main_stream): copy_stream.wait_stream(main_stream) @@ -81,7 +81,7 @@ class AsyncPoolingOutput(AsyncModelRunnerOutput): self.model_runner_output = model_runner_output self.pooler_output = pooler_output self.is_valid = is_valid - self.copy_event = torch.Event() + self.copy_event = torch.cuda.Event() with stream(copy_stream, main_stream): copy_stream.wait_stream(main_stream) diff --git a/vllm/v1/worker/gpu/pp_utils.py b/vllm/v1/worker/gpu/pp_utils.py index 00ff95b6da..9f5d4c2d80 100644 --- a/vllm/v1/worker/gpu/pp_utils.py +++ b/vllm/v1/worker/gpu/pp_utils.py @@ -17,7 +17,7 @@ from vllm.v1.worker.gpu.input_batch import InputBatch class PendingRecv: """Per-step slot data for a deferred postprocess on the main stream.""" - event: torch.Event + event: torch.cuda.Event sampled_tokens: torch.Tensor # [num_reqs, max_sample_len] num_sampled: torch.Tensor # [num_reqs] diff --git a/vllm/v1/worker/gpu/spec_decode/utils.py b/vllm/v1/worker/gpu/spec_decode/utils.py index 37ca166593..89d46a7582 100644 --- a/vllm/v1/worker/gpu/spec_decode/utils.py +++ b/vllm/v1/worker/gpu/spec_decode/utils.py @@ -12,7 +12,7 @@ class DraftTokensHandler: def __init__(self, device: torch.device | None = None): self.device = device self.copy_stream = torch.cuda.Stream(device) - self.copy_event = torch.Event() + self.copy_event = torch.cuda.Event() self.req_ids: list[str] = [] self.draft_tokens_np: np.ndarray | None = None diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index f83ed8a8ac..5a267a8bb8 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -860,7 +860,7 @@ class GPUModelRunner( # N-gram GPU path: async D2H buffer/event for per-request valid draft counts. self._num_valid_draft_tokens: torch.Tensor | None = None self._num_valid_draft_tokens_cpu: torch.Tensor | None = None - self._num_valid_draft_tokens_event: torch.Event | None = None + self._num_valid_draft_tokens_event: torch.cuda.Event | None = None self._num_valid_draft_tokens_copy_stream: torch.cuda.Stream | None = None if ( self.speculative_config is not None @@ -869,7 +869,7 @@ class GPUModelRunner( self._num_valid_draft_tokens_cpu = torch.empty( self.max_num_reqs, dtype=torch.int32, pin_memory=PIN_MEMORY ) - self._num_valid_draft_tokens_event = torch.Event() + self._num_valid_draft_tokens_event = torch.cuda.Event() self._num_valid_draft_tokens_copy_stream = torch.cuda.Stream() self._draft_token_req_ids: list[str] | None = None diff --git a/vllm/v1/worker/xpu_model_runner.py b/vllm/v1/worker/xpu_model_runner.py index 52647afa0a..82f129b7e6 100644 --- a/vllm/v1/worker/xpu_model_runner.py +++ b/vllm/v1/worker/xpu_model_runner.py @@ -49,6 +49,7 @@ def _torch_cuda_wrapper(): torch.cuda.current_stream = partial(torch.xpu.current_stream) torch.cuda.stream = partial(torch.xpu.stream) torch.cuda.set_stream = partial(torch.xpu.set_stream) + torch.cuda.Event = partial(torch.xpu.Event) if supports_xpu_graph(): torch.cuda.graph = partial(torch.xpu.graph) torch.cuda.CUDAGraph = torch.xpu.XPUGraph