Revert "[Misc] Remove use of CUDA_VISIBLE_DEVICES for device selectio… (#27502)
This commit is contained in:
@@ -991,14 +991,11 @@ class NixlConnectorWorker:
|
||||
# Enable different block lengths for different layers when MLA is used.
|
||||
self.block_len_per_layer = list[int]()
|
||||
self.slot_size_per_layer = list[int]() # HD bytes in kv terms
|
||||
self.device_id = self.tp_rank
|
||||
for layer_name, cache_or_caches in xfer_buffers.items():
|
||||
cache_list = cache_or_caches if split_k_and_v else [cache_or_caches]
|
||||
|
||||
for cache in cache_list:
|
||||
base_addr = cache.data_ptr()
|
||||
if not self.use_host_buffer and current_platform.is_cuda_alike():
|
||||
self.device_id = cache.device.index
|
||||
if base_addr in seen_base_addresses:
|
||||
continue
|
||||
|
||||
@@ -1026,7 +1023,7 @@ class NixlConnectorWorker:
|
||||
"All kv cache tensors must have the same size"
|
||||
)
|
||||
caches_data.append(
|
||||
(base_addr, curr_tensor_size_bytes, self.device_id, "")
|
||||
(base_addr, curr_tensor_size_bytes, self.tp_rank, "")
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
@@ -1073,7 +1070,7 @@ class NixlConnectorWorker:
|
||||
block_offset = block_id * self.block_len_per_layer[i]
|
||||
addr = base_addr + block_offset
|
||||
# (addr, len, device id)
|
||||
blocks_data.append((addr, kv_block_len, self.device_id))
|
||||
blocks_data.append((addr, kv_block_len, self.tp_rank))
|
||||
|
||||
if self._use_flashinfer:
|
||||
# Separate and interleave K/V regions to maintain the same
|
||||
@@ -1084,13 +1081,12 @@ class NixlConnectorWorker:
|
||||
addr = base_addr + block_offset
|
||||
# Register addresses for V cache (K registered first).
|
||||
v_addr = addr + kv_block_len
|
||||
blocks_data.append((v_addr, kv_block_len, self.device_id))
|
||||
blocks_data.append((v_addr, kv_block_len, self.tp_rank))
|
||||
logger.debug(
|
||||
"Created %s blocks for src engine %s and rank %s on device id %s",
|
||||
"Created %s blocks for src engine %s and rank %s",
|
||||
len(blocks_data),
|
||||
self.engine_id,
|
||||
self.tp_rank,
|
||||
self.device_id,
|
||||
)
|
||||
|
||||
descs = self.nixl_wrapper.get_xfer_descs(blocks_data, self.nixl_memory_type)
|
||||
|
||||
Reference in New Issue
Block a user