[Hardware] Replace torch.cuda.synchronize() api with torch.accelerator.synchronize (#36085)

Signed-off-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
Kunshang Ji
2026-03-05 10:36:39 +00:00
committed by GitHub
parent 0bfa229bf1
commit 66a2209645
59 changed files with 158 additions and 161 deletions
@@ -100,7 +100,7 @@ def test_dynamic_shapes_compilation(
del model
gc.collect()
torch.accelerator.empty_cache()
torch.cuda.synchronize()
torch.accelerator.synchronize()
print("GPU memory cleared")
+3 -3
View File
@@ -32,7 +32,7 @@ pointers = CustomAllreduce.create_shared_buffer(buffer_size_in_bytes)
print(f"Rank {rank} has pointers {pointers}")
dist.barrier()
torch.cuda.synchronize()
torch.accelerator.synchronize()
if rank == 0:
# the first rank tries to write to all buffers
@@ -41,7 +41,7 @@ if rank == 0:
lib.cudaMemset(pointer, byte_value, buffer_size_in_bytes)
dist.barrier()
torch.cuda.synchronize()
torch.accelerator.synchronize()
host_data = (ctypes.c_char * buffer_size_in_bytes)()
@@ -59,6 +59,6 @@ for p in pointers:
print(f"Rank {rank} verified all buffers")
dist.barrier()
torch.cuda.synchronize()
torch.accelerator.synchronize()
CustomAllreduce.free_shared_buffer(pointers)
+2 -2
View File
@@ -48,7 +48,7 @@ def graph_allreduce(
data = torch.zeros(1)
data = data.to(device=device)
torch.distributed.all_reduce(data, group=group)
torch.cuda.synchronize()
torch.accelerator.synchronize()
del data
# we use the first group to communicate once
@@ -68,7 +68,7 @@ def graph_allreduce(
inp2 = torch.randint(
1, 16, (sz,), dtype=dtype, device=torch.cuda.current_device()
)
torch.cuda.synchronize()
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=graph_capture_context.stream):
for i in range(num_communication):
+15 -15
View File
@@ -68,7 +68,7 @@ def worker_fn():
)
tensor = torch.ones(16, 1024, 1024, dtype=torch.float32).cuda(pynccl_comm.rank)
tensor = pynccl_comm.all_reduce(tensor)
torch.cuda.synchronize()
torch.accelerator.synchronize()
assert torch.all(tensor == pynccl_comm.world_size).cpu().item()
@@ -93,11 +93,11 @@ def multiple_allreduce_worker_fn():
if torch.distributed.get_rank() in [0, 1]:
tensor = pynccl_comm.all_reduce(tensor)
tensor = pynccl_comm.all_reduce(tensor)
torch.cuda.synchronize()
torch.accelerator.synchronize()
assert torch.all(tensor == 4).cpu().item()
else:
tensor = pynccl_comm.all_reduce(tensor)
torch.cuda.synchronize()
torch.accelerator.synchronize()
assert torch.all(tensor == 2).cpu().item()
@@ -121,11 +121,11 @@ def multiple_allreduce_with_vllm_worker_fn():
if torch.distributed.get_rank() in [0, 1]:
tensor = tensor_model_parallel_all_reduce(tensor)
tensor = tensor_model_parallel_all_reduce(tensor)
torch.cuda.synchronize()
torch.accelerator.synchronize()
assert torch.all(tensor == 4).cpu().item()
else:
tensor = tensor_model_parallel_all_reduce(tensor)
torch.cuda.synchronize()
torch.accelerator.synchronize()
assert torch.all(tensor == 2).cpu().item()
@@ -147,12 +147,12 @@ def worker_fn_with_cudagraph():
)
# run something in the default stream to initialize torch engine
a = torch.ones((4, 4), device=f"cuda:{pynccl_comm.rank}")
torch.cuda.synchronize()
torch.accelerator.synchronize()
with torch.cuda.graph(graph):
a_out = pynccl_comm.all_reduce(a)
torch.cuda.synchronize()
torch.accelerator.synchronize()
graph.replay()
torch.cuda.synchronize()
torch.accelerator.synchronize()
assert torch.all(a_out == pynccl_comm.world_size).cpu().item()
@@ -180,7 +180,7 @@ def all_gather_worker_fn():
).to(device)
pynccl_comm.all_gather(result, tensor)
torch.cuda.synchronize()
torch.accelerator.synchronize()
torch.testing.assert_close(result, expected, rtol=1e-5, atol=1e-8)
@@ -215,7 +215,7 @@ def all_gatherv_worker_fn():
).to(device)
pynccl_comm.all_gatherv(result, tensor, sizes=sizes)
torch.cuda.synchronize()
torch.accelerator.synchronize()
torch.testing.assert_close(result, expected, rtol=1e-5, atol=1e-8)
@@ -255,7 +255,7 @@ def reduce_scatter_worker_fn():
).to(device)
pynccl_comm.reduce_scatter(result, tensor)
torch.cuda.synchronize()
torch.accelerator.synchronize()
torch.testing.assert_close(result, expected, rtol=1e-5, atol=1e-8)
@@ -293,7 +293,7 @@ def reduce_scatterv_worker_fn():
expected = sum(tensor[start:end] for tensor in all_tensors).to(device)
pynccl_comm.reduce_scatterv(result, tensor, sizes=sizes)
torch.cuda.synchronize()
torch.accelerator.synchronize()
torch.testing.assert_close(result, expected, rtol=1e-5, atol=1e-8)
@@ -325,7 +325,7 @@ def send_recv_worker_fn():
pynccl_comm.send(tensor, dst=(pynccl_comm.rank + 1) % pynccl_comm.world_size)
else:
pynccl_comm.recv(tensor, src=(pynccl_comm.rank - 1) % pynccl_comm.world_size)
torch.cuda.synchronize()
torch.accelerator.synchronize()
assert torch.all(tensor == 1).cpu().item()
@@ -355,7 +355,7 @@ def multiple_send_recv_worker_fn():
pynccl_comm.send(tensor, dst=(pynccl_comm.rank + 1) % pynccl_comm.world_size)
else:
pynccl_comm.recv(tensor, src=(pynccl_comm.rank - 1) % pynccl_comm.world_size)
torch.cuda.synchronize()
torch.accelerator.synchronize()
if torch.distributed.get_rank() in [0, 2]:
assert torch.all(tensor == 1).cpu().item()
else:
@@ -396,7 +396,7 @@ def broadcast_worker_fn():
pynccl_comm.broadcast(recv_tensors[i], src=i)
# the broadcast op might be launched in a different stream
# need to synchronize to make sure the tensor is ready
torch.cuda.synchronize()
torch.accelerator.synchronize()
assert torch.all(recv_tensors[i] == i).cpu().item()
+2 -2
View File
@@ -52,7 +52,7 @@ def graph_quickreduce(
data = torch.zeros(1)
data = data.to(device=device)
torch.distributed.all_reduce(data, group=group)
torch.cuda.synchronize()
torch.accelerator.synchronize()
del data
# we use the first group to communicate once
@@ -71,7 +71,7 @@ def graph_quickreduce(
inp2 = torch.randint(
-23, 1, (sz,), dtype=dtype, device=torch.cuda.current_device()
)
torch.cuda.synchronize()
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=graph_capture_context.stream):
for _ in range(num_communication):
+2 -2
View File
@@ -79,11 +79,11 @@ def gpu_worker(rank, WORLD_SIZE, port1, port2):
data = torch.tensor([rank]).cuda()
pynccl1.all_reduce(data)
pg1.barrier()
torch.cuda.synchronize()
torch.accelerator.synchronize()
if rank <= 2:
pynccl2.all_reduce(data)
pg2.barrier()
torch.cuda.synchronize()
torch.accelerator.synchronize()
item = data[0].item()
print(f"rank: {rank}, item: {item}")
if rank == 3:
+4 -4
View File
@@ -251,7 +251,7 @@ def trainer_broadcast_tensor(
dtype = getattr(torch, tensor_dtype)
tensor_to_send = torch.ones(tensor_shape, dtype=dtype, device="cuda:0")
comm.broadcast(tensor_to_send, src=0, stream=torch.cuda.current_stream())
torch.cuda.synchronize()
torch.accelerator.synchronize()
return True
@@ -309,7 +309,7 @@ def inference_receive_tensor(
shapes=[tensor_shape],
)
engine.receive_weights(update_info, noop_load_weights)
torch.cuda.synchronize()
torch.accelerator.synchronize()
# Verify we received the tensor
success = False
@@ -630,7 +630,7 @@ class TrainerActor:
ipc_handle = reduce_tensor(self.tensor)
gpu_uuid = get_physical_gpu_id(0)
torch.cuda.synchronize()
torch.accelerator.synchronize()
self.ipc_handle_dict = {
"ipc_handle": ipc_handle,
@@ -704,7 +704,7 @@ def inference_receive_ipc_tensor(
update_info = engine.parse_update_info(update_dict)
engine.receive_weights(update_info, noop_load_weights)
torch.cuda.synchronize()
torch.accelerator.synchronize()
# Verify we received the tensor
success = False
@@ -165,7 +165,7 @@ def test_merge_attn_states(
suffix_lse_torch,
output_lse_torch,
)
torch.cuda.synchronize()
torch.accelerator.synchronize()
for _ in range(repeat_times):
start.record()
@@ -178,7 +178,7 @@ def test_merge_attn_states(
output_lse_torch,
)
end.record()
torch.cuda.synchronize()
torch.accelerator.synchronize()
total_time_torch_kernel += start.elapsed_time(end)
avg_time_torch_kernel = total_time_torch_kernel / repeat_times
@@ -200,7 +200,7 @@ def test_merge_attn_states(
suffix_lse,
output_lse_ref_triton,
)
torch.cuda.synchronize()
torch.accelerator.synchronize()
for _ in range(repeat_times):
start.record()
@@ -213,7 +213,7 @@ def test_merge_attn_states(
output_lse_ref_triton,
)
end.record()
torch.cuda.synchronize()
torch.accelerator.synchronize()
total_time_triton_kernel += start.elapsed_time(end)
avg_time_triton_kernel = total_time_triton_kernel / repeat_times
@@ -232,7 +232,7 @@ def test_merge_attn_states(
suffix_lse,
output_lse_cuda,
)
torch.cuda.synchronize()
torch.accelerator.synchronize()
for _ in range(repeat_times):
start.record()
@@ -245,7 +245,7 @@ def test_merge_attn_states(
output_lse_cuda,
)
end.record()
torch.cuda.synchronize()
torch.accelerator.synchronize()
total_time_cuda_kernel += start.elapsed_time(end)
avg_time_cuda_kernel = total_time_cuda_kernel / repeat_times
@@ -239,7 +239,7 @@ def test_contexted_kv_attention(
v_scale,
sliding_window=sliding_window,
)
torch.cuda.synchronize()
torch.accelerator.synchronize()
start_time = time.time()
op(
query,
@@ -258,7 +258,7 @@ def test_contexted_kv_attention(
v_scale,
sliding_window=sliding_window,
)
torch.cuda.synchronize()
torch.accelerator.synchronize()
end_time = time.time()
print(f"triton Time: {(end_time - start_time) * 1000:.2f} ms")
@@ -298,7 +298,7 @@ def test_contexted_kv_attention(
dropout_p=0.0,
scale=scale,
)
torch.cuda.synchronize()
torch.accelerator.synchronize()
start_time = time.time()
output_ref = F.scaled_dot_product_attention(
query_sdpa,
@@ -308,7 +308,7 @@ def test_contexted_kv_attention(
dropout_p=0.0,
scale=scale,
)
torch.cuda.synchronize()
torch.accelerator.synchronize()
end_time = time.time()
print(f"PyTorch SDPA Time: {(end_time - start_time) * 1000:.2f} ms")
@@ -482,7 +482,7 @@ def test_contexted_kv_attention_alibi(
v_scale,
alibi_slopes=alibi_slopes,
)
torch.cuda.synchronize()
torch.accelerator.synchronize()
start_time = time.time()
op(
query,
@@ -501,7 +501,7 @@ def test_contexted_kv_attention_alibi(
v_scale,
alibi_slopes=alibi_slopes,
)
torch.cuda.synchronize()
torch.accelerator.synchronize()
end_time = time.time()
print(f"triton Time: {(end_time - start_time) * 1000:.2f} ms")
scale = float(1.0 / (head_size**0.5))
@@ -517,7 +517,7 @@ def test_contexted_kv_attention_alibi(
output_ref = torch.empty_like(output)
torch.cuda.synchronize()
torch.accelerator.synchronize()
start_time = time.time()
query_start = 0
@@ -572,7 +572,7 @@ def test_contexted_kv_attention_alibi(
query_start = query_end
key_start = key_end
torch.cuda.synchronize()
torch.accelerator.synchronize()
end_time = time.time()
print(f"PyTorch SDPA Time: {(end_time - start_time) * 1000:.2f} ms")
atol = 1e-3 if "fp8" in kv_cache_dtype else 1e-6
+1 -1
View File
@@ -127,7 +127,7 @@ def test_fused_rms_norm_quant(
out_quant, x_unfused.contiguous(), quant_scale_t
)
torch.cuda.synchronize()
torch.accelerator.synchronize()
torch.testing.assert_close(residual_fused, residual, atol=1e-2, rtol=1e-2)
opcheck(
torch.ops._C.fused_add_rms_norm_static_fp8_quant,
@@ -34,7 +34,7 @@ def do_profile(
record_shapes=True,
) as tprof:
fn(**fn_kwargs)
torch.cuda.synchronize(torch.cuda.current_device())
torch.accelerator.synchronize(torch.cuda.current_device())
# TODO (varun): Add a descriptive trace file name
tprof.export_chrome_trace(
+2 -2
View File
@@ -318,8 +318,8 @@ def test_w8a8_block_fp8_deep_gemm_fused_moe(M, N, K, E, topk, seed, monkeypatch)
out = deep_gemm_moe_fp8_fn(
a, w1, w2, w1_s, w2_s, topk_weights, topk_ids
)
torch.cuda.synchronize()
torch.accelerator.synchronize()
graph.replay()
torch.cuda.synchronize()
torch.accelerator.synchronize()
torch.testing.assert_close(out, ref_out, atol=0.035, rtol=0.035)
+2 -2
View File
@@ -399,9 +399,9 @@ def test_cutlass_moe_8_bit_cuda_graph(
mt, topk_weights, topk_ids, per_act_token, per_out_ch
)
torch.cuda.synchronize()
torch.accelerator.synchronize()
graph.replay()
torch.cuda.synchronize()
torch.accelerator.synchronize()
torch.testing.assert_close(triton_output, cutlass_output, atol=9e-2, rtol=1e-2)
+3 -3
View File
@@ -272,9 +272,9 @@ def run_moe_test(
global_num_experts=global_num_experts,
expert_map=expert_map,
)
torch.cuda.synchronize()
torch.accelerator.synchronize()
graph.replay()
torch.cuda.synchronize()
torch.accelerator.synchronize()
torch.testing.assert_close(test_output, baseline_output, atol=atol, rtol=rtol)
@@ -768,7 +768,7 @@ def test_mixtral_moe(
F.pad(vllm_moe.experts.w2_weight, (0, 128), "constant", 0)[..., 0:-128],
requires_grad=False,
)
torch.cuda.synchronize()
torch.accelerator.synchronize()
torch.accelerator.empty_cache()
# FIXME (zyongye) fix this after we move self.kernel
@@ -122,7 +122,7 @@ def test_gptq_allspark_gemm_ampere(mnk_factors, group_size, has_zp, dtype):
)
output_ref = torch.matmul(input, w_ref)
torch.cuda.synchronize()
torch.accelerator.synchronize()
max_diff = compute_max_diff(output, output_ref)
assert max_diff < 0.04
@@ -269,7 +269,7 @@ def test_cutlass_w4a8_moe_mm_end_to_end(shape, random_zero):
setup.c_strides,
setup.group_scale_strides,
)
torch.cuda.synchronize()
torch.accelerator.synchronize()
out_ref = compute_moe_reference_output(setup)
torch.testing.assert_close(setup.out, out_ref, rtol=1e-2, atol=1e-2)
@@ -260,7 +260,7 @@ def test_gptq_marlin_repack(
marlin_q_w_2 = ops.gptq_marlin_repack(
q_w_gptq, sort_indices, size_k, size_n, quant_type.size_bits, is_a_8bit
)
torch.cuda.synchronize()
torch.accelerator.synchronize()
torch.testing.assert_close(marlin_q_w_1, marlin_q_w_2)
@@ -308,7 +308,7 @@ def test_awq_marlin_repack(k_chunk, n_chunk, quant_type, is_a_8bit, nk_factors):
marlin_q_w_2 = ops.awq_marlin_repack(
q_w_awq, size_k, size_n, quant_type.size_bits, is_a_8bit
)
torch.cuda.synchronize()
torch.accelerator.synchronize()
torch.testing.assert_close(marlin_q_w_1, marlin_q_w_2)
@@ -564,7 +564,7 @@ def test_marlin_gemm_subset_input():
)
output_ref = torch.matmul(a_input, w_ref)
torch.cuda.synchronize()
torch.accelerator.synchronize()
max_diff = compute_max_diff(output, output_ref)
@@ -613,7 +613,7 @@ def test_marlin_gemm_with_bias(size_m):
)
output_ref = torch.matmul(a_input, w_ref) + b_bias.view(1, -1)
torch.cuda.synchronize()
torch.accelerator.synchronize()
max_diff = compute_max_diff(output, output_ref)
+1 -1
View File
@@ -57,7 +57,7 @@ def test_gather_cache_oob():
seq_starts,
)
torch.cuda.synchronize()
torch.accelerator.synchronize()
assert True
+1 -1
View File
@@ -219,7 +219,7 @@ def _run_top_k_per_row_decode_test(
top_k,
)
torch.cuda.synchronize()
torch.accelerator.synchronize()
# Run reference implementation
torch_indices = torch.empty((num_rows, top_k), dtype=torch.int32, device="cuda")
@@ -195,4 +195,4 @@ def test_models(
# unit tests. On ROCm, when using AITER
# the memory might not be deallocated completely
# before running the next test case
torch.cuda.synchronize()
torch.accelerator.synchronize()
@@ -196,7 +196,7 @@ def test_compressed_tensors_w8a8_logprobs(
)
if current_platform.is_rocm():
torch.cuda.synchronize()
torch.accelerator.synchronize()
def test_compressed_tensors_no_enforce_eager(vllm_runner):