[Hardware] Replace torch.cuda.synchronize() api with torch.accelerator.synchronize (#36085)
Signed-off-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
@@ -32,7 +32,7 @@ pointers = CustomAllreduce.create_shared_buffer(buffer_size_in_bytes)
|
||||
print(f"Rank {rank} has pointers {pointers}")
|
||||
|
||||
dist.barrier()
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
if rank == 0:
|
||||
# the first rank tries to write to all buffers
|
||||
@@ -41,7 +41,7 @@ if rank == 0:
|
||||
lib.cudaMemset(pointer, byte_value, buffer_size_in_bytes)
|
||||
|
||||
dist.barrier()
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
host_data = (ctypes.c_char * buffer_size_in_bytes)()
|
||||
|
||||
@@ -59,6 +59,6 @@ for p in pointers:
|
||||
print(f"Rank {rank} verified all buffers")
|
||||
|
||||
dist.barrier()
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
CustomAllreduce.free_shared_buffer(pointers)
|
||||
|
||||
@@ -48,7 +48,7 @@ def graph_allreduce(
|
||||
data = torch.zeros(1)
|
||||
data = data.to(device=device)
|
||||
torch.distributed.all_reduce(data, group=group)
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
del data
|
||||
|
||||
# we use the first group to communicate once
|
||||
@@ -68,7 +68,7 @@ def graph_allreduce(
|
||||
inp2 = torch.randint(
|
||||
1, 16, (sz,), dtype=dtype, device=torch.cuda.current_device()
|
||||
)
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=graph_capture_context.stream):
|
||||
for i in range(num_communication):
|
||||
|
||||
@@ -68,7 +68,7 @@ def worker_fn():
|
||||
)
|
||||
tensor = torch.ones(16, 1024, 1024, dtype=torch.float32).cuda(pynccl_comm.rank)
|
||||
tensor = pynccl_comm.all_reduce(tensor)
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.all(tensor == pynccl_comm.world_size).cpu().item()
|
||||
|
||||
|
||||
@@ -93,11 +93,11 @@ def multiple_allreduce_worker_fn():
|
||||
if torch.distributed.get_rank() in [0, 1]:
|
||||
tensor = pynccl_comm.all_reduce(tensor)
|
||||
tensor = pynccl_comm.all_reduce(tensor)
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.all(tensor == 4).cpu().item()
|
||||
else:
|
||||
tensor = pynccl_comm.all_reduce(tensor)
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.all(tensor == 2).cpu().item()
|
||||
|
||||
|
||||
@@ -121,11 +121,11 @@ def multiple_allreduce_with_vllm_worker_fn():
|
||||
if torch.distributed.get_rank() in [0, 1]:
|
||||
tensor = tensor_model_parallel_all_reduce(tensor)
|
||||
tensor = tensor_model_parallel_all_reduce(tensor)
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.all(tensor == 4).cpu().item()
|
||||
else:
|
||||
tensor = tensor_model_parallel_all_reduce(tensor)
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.all(tensor == 2).cpu().item()
|
||||
|
||||
|
||||
@@ -147,12 +147,12 @@ def worker_fn_with_cudagraph():
|
||||
)
|
||||
# run something in the default stream to initialize torch engine
|
||||
a = torch.ones((4, 4), device=f"cuda:{pynccl_comm.rank}")
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
with torch.cuda.graph(graph):
|
||||
a_out = pynccl_comm.all_reduce(a)
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
graph.replay()
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.all(a_out == pynccl_comm.world_size).cpu().item()
|
||||
|
||||
|
||||
@@ -180,7 +180,7 @@ def all_gather_worker_fn():
|
||||
).to(device)
|
||||
|
||||
pynccl_comm.all_gather(result, tensor)
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
torch.testing.assert_close(result, expected, rtol=1e-5, atol=1e-8)
|
||||
|
||||
|
||||
@@ -215,7 +215,7 @@ def all_gatherv_worker_fn():
|
||||
).to(device)
|
||||
|
||||
pynccl_comm.all_gatherv(result, tensor, sizes=sizes)
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
torch.testing.assert_close(result, expected, rtol=1e-5, atol=1e-8)
|
||||
|
||||
|
||||
@@ -255,7 +255,7 @@ def reduce_scatter_worker_fn():
|
||||
).to(device)
|
||||
|
||||
pynccl_comm.reduce_scatter(result, tensor)
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
torch.testing.assert_close(result, expected, rtol=1e-5, atol=1e-8)
|
||||
|
||||
|
||||
@@ -293,7 +293,7 @@ def reduce_scatterv_worker_fn():
|
||||
expected = sum(tensor[start:end] for tensor in all_tensors).to(device)
|
||||
|
||||
pynccl_comm.reduce_scatterv(result, tensor, sizes=sizes)
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
torch.testing.assert_close(result, expected, rtol=1e-5, atol=1e-8)
|
||||
|
||||
|
||||
@@ -325,7 +325,7 @@ def send_recv_worker_fn():
|
||||
pynccl_comm.send(tensor, dst=(pynccl_comm.rank + 1) % pynccl_comm.world_size)
|
||||
else:
|
||||
pynccl_comm.recv(tensor, src=(pynccl_comm.rank - 1) % pynccl_comm.world_size)
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.all(tensor == 1).cpu().item()
|
||||
|
||||
|
||||
@@ -355,7 +355,7 @@ def multiple_send_recv_worker_fn():
|
||||
pynccl_comm.send(tensor, dst=(pynccl_comm.rank + 1) % pynccl_comm.world_size)
|
||||
else:
|
||||
pynccl_comm.recv(tensor, src=(pynccl_comm.rank - 1) % pynccl_comm.world_size)
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
if torch.distributed.get_rank() in [0, 2]:
|
||||
assert torch.all(tensor == 1).cpu().item()
|
||||
else:
|
||||
@@ -396,7 +396,7 @@ def broadcast_worker_fn():
|
||||
pynccl_comm.broadcast(recv_tensors[i], src=i)
|
||||
# the broadcast op might be launched in a different stream
|
||||
# need to synchronize to make sure the tensor is ready
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.all(recv_tensors[i] == i).cpu().item()
|
||||
|
||||
|
||||
|
||||
@@ -52,7 +52,7 @@ def graph_quickreduce(
|
||||
data = torch.zeros(1)
|
||||
data = data.to(device=device)
|
||||
torch.distributed.all_reduce(data, group=group)
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
del data
|
||||
|
||||
# we use the first group to communicate once
|
||||
@@ -71,7 +71,7 @@ def graph_quickreduce(
|
||||
inp2 = torch.randint(
|
||||
-23, 1, (sz,), dtype=dtype, device=torch.cuda.current_device()
|
||||
)
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=graph_capture_context.stream):
|
||||
for _ in range(num_communication):
|
||||
|
||||
@@ -79,11 +79,11 @@ def gpu_worker(rank, WORLD_SIZE, port1, port2):
|
||||
data = torch.tensor([rank]).cuda()
|
||||
pynccl1.all_reduce(data)
|
||||
pg1.barrier()
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
if rank <= 2:
|
||||
pynccl2.all_reduce(data)
|
||||
pg2.barrier()
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
item = data[0].item()
|
||||
print(f"rank: {rank}, item: {item}")
|
||||
if rank == 3:
|
||||
|
||||
@@ -251,7 +251,7 @@ def trainer_broadcast_tensor(
|
||||
dtype = getattr(torch, tensor_dtype)
|
||||
tensor_to_send = torch.ones(tensor_shape, dtype=dtype, device="cuda:0")
|
||||
comm.broadcast(tensor_to_send, src=0, stream=torch.cuda.current_stream())
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
return True
|
||||
|
||||
@@ -309,7 +309,7 @@ def inference_receive_tensor(
|
||||
shapes=[tensor_shape],
|
||||
)
|
||||
engine.receive_weights(update_info, noop_load_weights)
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
# Verify we received the tensor
|
||||
success = False
|
||||
@@ -630,7 +630,7 @@ class TrainerActor:
|
||||
ipc_handle = reduce_tensor(self.tensor)
|
||||
gpu_uuid = get_physical_gpu_id(0)
|
||||
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
self.ipc_handle_dict = {
|
||||
"ipc_handle": ipc_handle,
|
||||
@@ -704,7 +704,7 @@ def inference_receive_ipc_tensor(
|
||||
|
||||
update_info = engine.parse_update_info(update_dict)
|
||||
engine.receive_weights(update_info, noop_load_weights)
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
# Verify we received the tensor
|
||||
success = False
|
||||
|
||||
Reference in New Issue
Block a user