[Hardware] Replace torch.cuda.device_count/current_device/set_device API (#36145)

Signed-off-by: Kunshang Ji <jikunshang95@gmail.com>
Signed-off-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
Kunshang Ji
2026-03-12 07:57:47 -07:00
committed by GitHub
parent 2e693f48e7
commit 53ec16a705
89 changed files with 254 additions and 219 deletions
@@ -190,7 +190,7 @@ class StaticSinkAttention(Attention, CustomOp):
sink_kv_slot_mapping = torch.arange(
self.block_size,
self.sink_len + self.block_size,
device=torch.cuda.current_device(),
device=torch.accelerator.current_device_index(),
dtype=torch.long,
)
triton_reshape_and_cache_flash_diffkv(
@@ -295,14 +295,17 @@ class DefaultMoERunner(MoERunner):
states_shape = (moe.max_num_tokens, self.moe_config.hidden_dim)
logits_shape = (moe.max_num_tokens, self.moe_config.num_logical_experts)
device = torch.accelerator.current_device_index()
self.batched_hidden_states = torch.zeros(
states_shape, dtype=moe.in_dtype, device=torch.cuda.current_device()
states_shape,
dtype=moe.in_dtype,
device=device,
)
self.batched_router_logits = torch.zeros(
logits_shape,
dtype=moe.router_logits_dtype,
device=torch.cuda.current_device(),
device=device,
)
def must_reduce_shared_expert_outputs(self) -> bool:
@@ -28,7 +28,7 @@ class TrtLlmGenExperts(mk.FusedMoEExpertsModular):
max_capture_size,
):
super().__init__(moe_config, quant_config)
self.device = torch.cuda.current_device()
self.device = torch.accelerator.current_device_index()
self.num_experts = moe_config.num_local_experts
self.gemm1_alpha = torch.tensor(
[1.702] * self.num_experts, dtype=torch.float32, device=self.device
+1 -1
View File
@@ -202,7 +202,7 @@ class RMSNorm(CustomOp):
# external Oink initialization work in this case.
else:
try:
device_index = torch.cuda.current_device()
device_index = torch.accelerator.current_device_index()
if _oink_ops.is_oink_available_for_device(device_index):
self._use_oink_rmsnorm = True
self._use_oink_fused_add_rmsnorm = (
@@ -36,7 +36,8 @@ class DualChunkRotaryEmbedding(CustomOp):
self.chunk_size = chunk_size
self.local_size = local_size
self.dtype = dtype
self.device = torch.device(f"cuda:{torch.cuda.current_device()}")
device_idx = torch.accelerator.current_device_index()
self.device = torch.device(f"cuda:{device_idx}")
(q_cache, qc_cache, k_cache, qc_no_clamp_cache, q_inter_cache) = (
self._compute_cos_sin_cache()
)