[Hardware] Replace torch.cuda.synchronize() api with torch.accelerator.synchronize (#36085)
Signed-off-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
@@ -217,7 +217,7 @@ class ElasticEPScalingExecutor:
|
||||
dp_group=standby_dp_group,
|
||||
expert_weights=model.expert_weights,
|
||||
)
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
def broadcast_expert_mapping(self) -> None:
|
||||
standby_dp_group = get_standby_dp_group()
|
||||
@@ -407,7 +407,7 @@ class ElasticEPScalingExecutor:
|
||||
reset_compile_wrapper(self.worker.model_runner.get_model())
|
||||
|
||||
gc.collect()
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
torch.accelerator.empty_cache()
|
||||
unlock_workspace()
|
||||
self.worker.compile_or_warm_up_model()
|
||||
@@ -446,7 +446,7 @@ class ElasticEPScalingExecutor:
|
||||
|
||||
eplb_state.rearrange(rank_mapping=rank_mapping)
|
||||
# NOTE(yongji): check whether we need to synchronize here
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
# reset expert_rearrangement_step to ensure all ranks are synchronized
|
||||
eplb_state.expert_rearrangement_step = 0
|
||||
eplb_state.num_valid_physical_experts = (
|
||||
@@ -491,7 +491,7 @@ class ElasticEPScalingExecutor:
|
||||
dp_group=dp_group,
|
||||
expert_weights=model.expert_weights,
|
||||
)
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
def receive_expert_mapping(self) -> tuple[torch.Tensor, int, int]:
|
||||
dp_group = get_dp_group()
|
||||
|
||||
@@ -622,7 +622,7 @@ def rearrange_expert_weights_inplace(
|
||||
|
||||
# NOTE(bowen): We need this synchronize to run, but I don't know why.
|
||||
# If you figure out the reason, please let me know -- thank you!
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
old_global_expert_indices_cpu = old_global_expert_indices.cpu().numpy()
|
||||
new_global_expert_indices_cpu = new_global_expert_indices.cpu().numpy()
|
||||
|
||||
@@ -77,7 +77,7 @@ class CutlassW4A8LinearKernel(MPLinearKernel):
|
||||
def transform_w_q(x):
|
||||
assert isinstance(x, BasevLLMParameter)
|
||||
convert_packed_uint4b8_to_signed_int4_inplace(x.data)
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
permute_param_layout_(x, input_dim=0, output_dim=1, packed_dim=0)
|
||||
x.data = ops.cutlass_encode_and_reorder_int4b(x.data.t().contiguous().t())
|
||||
return x
|
||||
|
||||
@@ -457,7 +457,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
else:
|
||||
self._dummy_pooler_run(hidden_states)
|
||||
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
del hidden_states, sample_hidden_states
|
||||
gc.collect()
|
||||
|
||||
@@ -525,7 +525,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# to trigger JIT compilation.
|
||||
if all("FLASHINFER" in b.get_name() for b in self.attn_backends.values()):
|
||||
self._dummy_run(self.max_num_tokens, skip_attn=False)
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
def finish_requests(self, scheduler_output: SchedulerOutput) -> None:
|
||||
finished_req_ids = scheduler_output.finished_req_ids
|
||||
|
||||
@@ -102,4 +102,4 @@ def warmup_kernels(model_runner: GPUModelRunner) -> None:
|
||||
cleanup_output.finished_req_ids = set(req_ids)
|
||||
model_runner.execute_model(cleanup_output)
|
||||
model_runner.kv_connector.set_disabled(False)
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
@@ -928,7 +928,7 @@ class GPUModelRunner(
|
||||
|
||||
# Note: used for model runner override.
|
||||
def _sync_device(self) -> None:
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
def _update_states(self, scheduler_output: "SchedulerOutput") -> None:
|
||||
"""Update the cached states and the persistent batch with the scheduler
|
||||
@@ -5345,7 +5345,7 @@ class GPUModelRunner(
|
||||
cudagraph_runtime_mode=runtime_mode,
|
||||
)
|
||||
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
end_free_gpu_memory = torch.cuda.mem_get_info()[0]
|
||||
|
||||
# Disable cudagraph capturing globally, so any unexpected cudagraph
|
||||
@@ -6266,13 +6266,13 @@ class GPUModelRunner(
|
||||
group_refs = group_lora_refs[current_item_idx : current_item_idx + num_items]
|
||||
group_request_ids = {req_id for req_id, _ in group_refs}
|
||||
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
start_time = time.perf_counter()
|
||||
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
torch.cuda.synchronize()
|
||||
torch.accelerator.synchronize()
|
||||
elapsed = time.perf_counter() - start_time
|
||||
|
||||
per_request_time = elapsed / max(len(group_request_ids), 1)
|
||||
|
||||
@@ -29,9 +29,6 @@ class XPUModelRunner(GPUModelRunner):
|
||||
# FIXME: To be verified.
|
||||
self.cascade_attn_enabled = False
|
||||
|
||||
def _sync_device(self) -> None:
|
||||
torch.xpu.synchronize()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _torch_cuda_wrapper():
|
||||
@@ -42,7 +39,6 @@ def _torch_cuda_wrapper():
|
||||
torch.cuda.current_stream = torch.xpu.current_stream
|
||||
torch.cuda.stream = torch.xpu.stream
|
||||
torch.cuda.mem_get_info = torch.xpu.mem_get_info
|
||||
torch.cuda.synchronize = torch.xpu.synchronize
|
||||
if supports_xpu_graph():
|
||||
torch.cuda.graph = torch.xpu.graph
|
||||
torch.cuda.CUDAGraph = torch.xpu.XPUGraph
|
||||
|
||||
Reference in New Issue
Block a user