[Platform] Replace torch.cuda.Event with torch.Event (#47140)
Signed-off-by: Kunshang Ji <kunshang.ji@intel.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
parent
fcaa84efa7
commit
e840f0d3f5
@@ -742,8 +742,8 @@ class EplbState:
|
||||
is_main_rank = ep_rank == 0
|
||||
if is_main_rank:
|
||||
if not self.is_async or is_profile:
|
||||
start_event = torch.cuda.Event(enable_timing=True)
|
||||
end_event = torch.cuda.Event(enable_timing=True)
|
||||
start_event = torch.Event(enable_timing=True)
|
||||
end_event = torch.Event(enable_timing=True)
|
||||
start_event.record()
|
||||
logger.info(
|
||||
"Rearranging experts %s %s...",
|
||||
|
||||
@@ -31,7 +31,7 @@ class CpuGpuEvent:
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._event = torch.cuda.Event()
|
||||
self._event = torch.Event()
|
||||
self._recorded = threading.Event()
|
||||
|
||||
def wait(self, stream: torch.cuda.Stream | None = None):
|
||||
@@ -56,7 +56,7 @@ class CpuGpuEvent:
|
||||
"CpuGpuEvent.record() called before the previous event was "
|
||||
"consumed by wait()"
|
||||
)
|
||||
self._event = torch.cuda.Event()
|
||||
self._event = torch.Event()
|
||||
self._event.record(stream)
|
||||
self._recorded.set()
|
||||
|
||||
|
||||
@@ -239,7 +239,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
# this event is complete the request is considered "done sending"
|
||||
# by get_finished; clients block on the per-file flock to wait for
|
||||
# the disk write itself.
|
||||
self._req_copy_events: dict[str, torch.cuda.Event] = {}
|
||||
self._req_copy_events: dict[str, torch.Event] = {}
|
||||
# req_ids reported as finished-generating by the scheduler,
|
||||
# accumulated across get_finished calls.
|
||||
self._accumulated_finished_req_ids: set[str] = set()
|
||||
@@ -320,7 +320,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
@staticmethod
|
||||
def _write_tensors(
|
||||
tensors: dict[str, torch.Tensor],
|
||||
event: torch.cuda.Event,
|
||||
event: torch.Event,
|
||||
filename: str,
|
||||
lock_fd: int | None,
|
||||
) -> None:
|
||||
@@ -375,7 +375,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
copy_stream = self._get_copy_stream()
|
||||
|
||||
# Ensure the copy stream sees all prior writes on the default stream.
|
||||
ready_event = torch.cuda.Event()
|
||||
ready_event = torch.Event()
|
||||
ready_event.record()
|
||||
copy_stream.wait_event(ready_event)
|
||||
|
||||
@@ -396,7 +396,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
pinned_hs.copy_(hidden_states_gpu, non_blocking=True)
|
||||
|
||||
# Record completion of this copy on the copy stream.
|
||||
copy_done = torch.cuda.Event()
|
||||
copy_done = torch.Event()
|
||||
copy_done.record(copy_stream)
|
||||
|
||||
# token_ids is already on CPU (created in request_finished).
|
||||
|
||||
@@ -221,7 +221,7 @@ class Hf3fsClient:
|
||||
|
||||
@wsynchronized()
|
||||
def batch_write(
|
||||
self, offsets: list[int], tensors: list[torch.Tensor], event: torch.cuda.Event
|
||||
self, offsets: list[int], tensors: list[torch.Tensor], event: torch.Event
|
||||
) -> list[int]:
|
||||
"""Write data from tensors to the file at specified offsets.
|
||||
|
||||
|
||||
@@ -133,7 +133,7 @@ class AsyncOperationManager:
|
||||
# CUDA streams for async operations
|
||||
self._save_stream = torch.cuda.Stream()
|
||||
self._load_stream = torch.cuda.Stream()
|
||||
self._save_event = torch.cuda.Event()
|
||||
self._save_event = torch.Event()
|
||||
|
||||
# Buffer allocators for data copying
|
||||
self._save_buffer_allocator = CopyBufferAllocator(
|
||||
@@ -171,7 +171,7 @@ class AsyncOperationManager:
|
||||
def submit_save_operation(self, request_id: str, block_ids, block_hashes) -> Future:
|
||||
"""Submit a save operation for async execution."""
|
||||
future: Future[Any] = Future()
|
||||
main_stream_event = torch.cuda.Event()
|
||||
main_stream_event = torch.Event()
|
||||
main_stream_event.record()
|
||||
task = (request_id, block_ids, block_hashes, future, main_stream_event)
|
||||
self._save_queue.put(task)
|
||||
@@ -304,7 +304,7 @@ class AsyncOperationManager:
|
||||
block_ids, buffers, "gather"
|
||||
)
|
||||
|
||||
save_stream_event = torch.cuda.Event()
|
||||
save_stream_event = torch.Event()
|
||||
save_stream_event.record(self._save_stream) # Record gather completion
|
||||
|
||||
# Step3: Write data in batches
|
||||
|
||||
@@ -75,7 +75,7 @@ class Hf3fsClient:
|
||||
return torch.frombuffer(buffer_data, dtype=dtype)
|
||||
|
||||
def batch_write(
|
||||
self, offsets: list[int], tensors: list[torch.Tensor], event: torch.cuda.Event
|
||||
self, offsets: list[int], tensors: list[torch.Tensor], event: torch.Event
|
||||
) -> list[int]:
|
||||
"""Write data from tensors to file at specified offsets."""
|
||||
results = []
|
||||
|
||||
+4
-4
@@ -430,7 +430,7 @@ class LMCacheMPWorkerAdapter:
|
||||
|
||||
@_lmcache_nvtx_annotate
|
||||
def submit_store_request(
|
||||
self, request_id: str, op: LoadStoreOp, event: torch.cuda.Event
|
||||
self, request_id: str, op: LoadStoreOp, event: torch.Event
|
||||
):
|
||||
"""
|
||||
Submit a KV cache store request to LMCache
|
||||
@@ -464,7 +464,7 @@ class LMCacheMPWorkerAdapter:
|
||||
|
||||
@_lmcache_nvtx_annotate
|
||||
def submit_retrieve_request(
|
||||
self, request_id: str, op: LoadStoreOp, event: torch.cuda.Event
|
||||
self, request_id: str, op: LoadStoreOp, event: torch.Event
|
||||
):
|
||||
"""
|
||||
Submit a KV cache retrieve request to LMCache
|
||||
@@ -501,7 +501,7 @@ class LMCacheMPWorkerAdapter:
|
||||
self,
|
||||
request_ids: list[str],
|
||||
ops: list[LoadStoreOp],
|
||||
event: torch.cuda.Event,
|
||||
event: torch.Event,
|
||||
):
|
||||
"""
|
||||
Submit a batched store request to LMCache
|
||||
@@ -550,7 +550,7 @@ class LMCacheMPWorkerAdapter:
|
||||
self,
|
||||
request_ids: list[str],
|
||||
ops: list[LoadStoreOp],
|
||||
event: torch.cuda.Event,
|
||||
event: torch.Event,
|
||||
):
|
||||
"""
|
||||
Submit a batched retrieve request to LMCache
|
||||
|
||||
@@ -589,7 +589,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
return
|
||||
|
||||
with torch.cuda.stream(torch.cuda.current_stream()):
|
||||
event = torch.cuda.Event(interprocess=True)
|
||||
event = torch.Event(interprocess=True)
|
||||
event.record()
|
||||
|
||||
self.worker_adapter.batched_submit_retrieve_requests(
|
||||
@@ -663,7 +663,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
return
|
||||
|
||||
with torch.cuda.stream(torch.cuda.current_stream()):
|
||||
event = torch.cuda.Event(interprocess=True)
|
||||
event = torch.Event(interprocess=True)
|
||||
event.record()
|
||||
|
||||
self.worker_adapter.batched_submit_store_requests(
|
||||
|
||||
@@ -323,7 +323,7 @@ class ReqMeta:
|
||||
can_save: bool | None = None
|
||||
load_spec: LoadSpec | None = None
|
||||
is_last_chunk: bool | None = None
|
||||
current_event: torch.cuda.Event | None = None
|
||||
current_event: torch.Event | None = None
|
||||
|
||||
token_ids: list[int] | None = None
|
||||
num_prompt_tokens: int | None = None
|
||||
|
||||
@@ -1357,7 +1357,7 @@ class MooncakeStoreWorker:
|
||||
current_event = None
|
||||
for request in meta.requests:
|
||||
if request.can_save:
|
||||
current_event = torch.cuda.Event()
|
||||
current_event = torch.Event()
|
||||
current_event.record()
|
||||
break
|
||||
|
||||
|
||||
@@ -56,7 +56,7 @@ class WriteTask:
|
||||
local_block_ids: list[int]
|
||||
remote_block_ids_hint: list[int] | None
|
||||
layer_name: str
|
||||
event: torch.cuda.Event
|
||||
event: torch.Event
|
||||
remote_notify_port: int
|
||||
remote_ip: str
|
||||
enqueue_time: float = field(default_factory=time.perf_counter)
|
||||
|
||||
@@ -1061,7 +1061,7 @@ class MoRIIOConnectorWorker:
|
||||
# when mori-io supports ibgda functionality
|
||||
|
||||
stream = torch.cuda.current_stream()
|
||||
event = torch.cuda.Event()
|
||||
event = torch.Event()
|
||||
event.record(stream)
|
||||
|
||||
task = WriteTask(
|
||||
|
||||
Reference in New Issue
Block a user