[Hardware] Replace torch.cuda.device_count/current_device/set_device API (#36145)
Signed-off-by: Kunshang Ji <jikunshang95@gmail.com> Signed-off-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
@@ -190,7 +190,7 @@ class StaticSinkAttention(Attention, CustomOp):
|
||||
sink_kv_slot_mapping = torch.arange(
|
||||
self.block_size,
|
||||
self.sink_len + self.block_size,
|
||||
device=torch.cuda.current_device(),
|
||||
device=torch.accelerator.current_device_index(),
|
||||
dtype=torch.long,
|
||||
)
|
||||
triton_reshape_and_cache_flash_diffkv(
|
||||
|
||||
@@ -295,14 +295,17 @@ class DefaultMoERunner(MoERunner):
|
||||
states_shape = (moe.max_num_tokens, self.moe_config.hidden_dim)
|
||||
logits_shape = (moe.max_num_tokens, self.moe_config.num_logical_experts)
|
||||
|
||||
device = torch.accelerator.current_device_index()
|
||||
self.batched_hidden_states = torch.zeros(
|
||||
states_shape, dtype=moe.in_dtype, device=torch.cuda.current_device()
|
||||
states_shape,
|
||||
dtype=moe.in_dtype,
|
||||
device=device,
|
||||
)
|
||||
|
||||
self.batched_router_logits = torch.zeros(
|
||||
logits_shape,
|
||||
dtype=moe.router_logits_dtype,
|
||||
device=torch.cuda.current_device(),
|
||||
device=device,
|
||||
)
|
||||
|
||||
def must_reduce_shared_expert_outputs(self) -> bool:
|
||||
|
||||
@@ -28,7 +28,7 @@ class TrtLlmGenExperts(mk.FusedMoEExpertsModular):
|
||||
max_capture_size,
|
||||
):
|
||||
super().__init__(moe_config, quant_config)
|
||||
self.device = torch.cuda.current_device()
|
||||
self.device = torch.accelerator.current_device_index()
|
||||
self.num_experts = moe_config.num_local_experts
|
||||
self.gemm1_alpha = torch.tensor(
|
||||
[1.702] * self.num_experts, dtype=torch.float32, device=self.device
|
||||
|
||||
@@ -202,7 +202,7 @@ class RMSNorm(CustomOp):
|
||||
# external Oink initialization work in this case.
|
||||
else:
|
||||
try:
|
||||
device_index = torch.cuda.current_device()
|
||||
device_index = torch.accelerator.current_device_index()
|
||||
if _oink_ops.is_oink_available_for_device(device_index):
|
||||
self._use_oink_rmsnorm = True
|
||||
self._use_oink_fused_add_rmsnorm = (
|
||||
|
||||
@@ -36,7 +36,8 @@ class DualChunkRotaryEmbedding(CustomOp):
|
||||
self.chunk_size = chunk_size
|
||||
self.local_size = local_size
|
||||
self.dtype = dtype
|
||||
self.device = torch.device(f"cuda:{torch.cuda.current_device()}")
|
||||
device_idx = torch.accelerator.current_device_index()
|
||||
self.device = torch.device(f"cuda:{device_idx}")
|
||||
(q_cache, qc_cache, k_cache, qc_no_clamp_cache, q_inter_cache) = (
|
||||
self._compute_cos_sin_cache()
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user