[Hardware] Replace torch.cuda.device_count/current_device/set_device API (#36145)

Signed-off-by: Kunshang Ji <jikunshang95@gmail.com>
Signed-off-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
Kunshang Ji
2026-03-12 07:57:47 -07:00
committed by GitHub
parent 2e693f48e7
commit 53ec16a705
89 changed files with 254 additions and 219 deletions
@@ -190,7 +190,7 @@ class StaticSinkAttention(Attention, CustomOp):
sink_kv_slot_mapping = torch.arange(
self.block_size,
self.sink_len + self.block_size,
device=torch.cuda.current_device(),
device=torch.accelerator.current_device_index(),
dtype=torch.long,
)
triton_reshape_and_cache_flash_diffkv(
@@ -295,14 +295,17 @@ class DefaultMoERunner(MoERunner):
states_shape = (moe.max_num_tokens, self.moe_config.hidden_dim)
logits_shape = (moe.max_num_tokens, self.moe_config.num_logical_experts)
device = torch.accelerator.current_device_index()
self.batched_hidden_states = torch.zeros(
states_shape, dtype=moe.in_dtype, device=torch.cuda.current_device()
states_shape,
dtype=moe.in_dtype,
device=device,
)
self.batched_router_logits = torch.zeros(
logits_shape,
dtype=moe.router_logits_dtype,
device=torch.cuda.current_device(),
device=device,
)
def must_reduce_shared_expert_outputs(self) -> bool:
@@ -28,7 +28,7 @@ class TrtLlmGenExperts(mk.FusedMoEExpertsModular):
max_capture_size,
):
super().__init__(moe_config, quant_config)
self.device = torch.cuda.current_device()
self.device = torch.accelerator.current_device_index()
self.num_experts = moe_config.num_local_experts
self.gemm1_alpha = torch.tensor(
[1.702] * self.num_experts, dtype=torch.float32, device=self.device
+1 -1
View File
@@ -202,7 +202,7 @@ class RMSNorm(CustomOp):
# external Oink initialization work in this case.
else:
try:
device_index = torch.cuda.current_device()
device_index = torch.accelerator.current_device_index()
if _oink_ops.is_oink_available_for_device(device_index):
self._use_oink_rmsnorm = True
self._use_oink_fused_add_rmsnorm = (
@@ -36,7 +36,8 @@ class DualChunkRotaryEmbedding(CustomOp):
self.chunk_size = chunk_size
self.local_size = local_size
self.dtype = dtype
self.device = torch.device(f"cuda:{torch.cuda.current_device()}")
device_idx = torch.accelerator.current_device_index()
self.device = torch.device(f"cuda:{device_idx}")
(q_cache, qc_cache, k_cache, qc_no_clamp_cache, q_inter_cache) = (
self._compute_cos_sin_cache()
)
@@ -539,6 +539,8 @@ def deserialize_tensorizer_model(
)
before_mem = get_mem_usage()
start = time.perf_counter()
device_index = torch.accelerator.current_device_index()
device_type = current_platform.device_type
with (
open_stream(
tensorizer_config.tensorizer_uri, mode="rb", **tensorizer_args.stream_kwargs
@@ -546,9 +548,7 @@ def deserialize_tensorizer_model(
TensorDeserializer(
stream,
dtype=tensorizer_config.dtype,
device=f"xpu:{torch.xpu.current_device()}"
if current_platform.is_xpu()
else f"cuda:{torch.cuda.current_device()}",
device=f"{device_type}:{device_index}",
**tensorizer_args.deserialization_kwargs,
) as deserializer,
):