[Bugfix][ROCm][Disagg] Fix MoRIIO shared KV memory region registration (#53698)

Signed-off-by: avininjamay8 <avpaul@amd.com>
Co-authored-by: avininjamay8 <avpaul@amd.com>
This commit is contained in:
Avinash Paul
2026-08-27 00:10:57 +08:00
committed by GitHub
co-authored by avininjamay8
parent c71f6f8a81
commit 8d301f075b
5 changed files with 97 additions and 31 deletions
@@ -341,7 +341,7 @@ def test_mixed_layers_compute_distinct_offsets_per_layer():
assert interleaved != indexer
def test_write_transfer_plan_caches_offsets_per_geometry():
def test_write_transfer_plan_caches_offsets_per_layer_geometry():
kv_caches = {
"dense0": torch.empty((8, 2, 4, 2, 3), dtype=torch.bfloat16),
"dense1": torch.empty((8, 2, 4, 2, 3), dtype=torch.bfloat16),
@@ -354,7 +354,12 @@ def test_write_transfer_plan_caches_offsets_per_geometry():
layer_name_to_local_kv_cache_metadata: dict[str, list[Any]]
def _compute_block_transfer_offsets(
self, layer_name, local_block_ids, remote_block_ids, remote_moriio_meta
self,
layer_name,
local_block_ids,
remote_block_ids,
remote_moriio_meta,
remote_engine_id=None,
):
calls.append(layer_name)
call_id = len(calls)
@@ -369,41 +374,26 @@ def test_write_transfer_plan_caches_offsets_per_geometry():
remote_meta = _remote_meta()
dense0_plan = writer._prepare_transfer_plan(
SimpleNamespace(
layer_name="dense0",
local_block_ids=[1, 3],
request_id="req",
transfer_id="xfer",
),
_write_task("dense0"),
request_info,
remote_meta,
)
dense1_plan = writer._prepare_transfer_plan(
SimpleNamespace(
layer_name="dense1",
local_block_ids=[1, 3],
request_id="req",
transfer_id="xfer",
),
_write_task("dense1"),
request_info,
remote_meta,
)
indexer_plan = writer._prepare_transfer_plan(
SimpleNamespace(
layer_name="indexer",
local_block_ids=[1, 3],
request_id="req",
transfer_id="xfer",
),
_write_task("indexer"),
request_info,
remote_meta,
)
assert calls == ["dense0", "indexer"]
assert calls == ["dense0", "dense1", "indexer"]
assert dense0_plan.transfer_local_offsets == [1]
assert dense1_plan.transfer_local_offsets == [1]
assert indexer_plan.transfer_local_offsets == [2]
assert len(request_info.transfer_offsets) == 2
assert dense1_plan.transfer_local_offsets == [2]
assert indexer_plan.transfer_local_offsets == [3]
assert len(request_info.transfer_offsets) == 3
def test_write_scheduler_deduplicates_layers_and_seals_expected_count():
@@ -41,6 +41,7 @@ Transfer = tuple[int, float]
EngineId = str
ReqId = str
TransferId = str
TransferOffsetsKey = tuple[str, tuple[int, ...], tuple[int, ...], torch.dtype]
class MoRIIOTransferAck(NamedTuple):
@@ -91,7 +92,7 @@ class RemoteAllocInfo:
completion_notified: bool = False
transfer_statuses: list[Any] = field(default_factory=list)
transfer_offsets: dict[
tuple[tuple[int, ...], tuple[int, ...], torch.dtype],
TransferOffsetsKey,
tuple[list[int], list[int], list[int]],
] = field(default_factory=dict)
@@ -1226,6 +1226,8 @@ class MoRIIOConnectorWorker:
# KV Caches and moriio tracking data.
self.kv_caches: dict[str, torch.Tensor] = {}
self.kv_layer_mr_offset: dict[str, int] = {}
self.layer_base_addr_index: dict[str, int] = {}
# Map of engine_id -> kv_caches_base_addr. For TP case, each local
# rank will still only pull from a single remote TP worker.
@@ -1695,6 +1697,50 @@ class MoRIIOConnectorWorker:
self.layer_to_spec,
)
def _build_shared_kv_mr(
self, kv_caches: dict[str, torch.Tensor]
) -> tuple[torch.Tensor, dict[str, int]]:
page_size = 4096
backing = next(iter(kv_caches.values())).view(torch.uint8)
nbytes = backing.untyped_storage().nbytes()
base = torch.as_strided(backing, (nbytes,), (1,), 0)
ptr = base.data_ptr()
slack = ptr % page_size
end = 0
for layer_name in kv_caches:
_, region_len = next(
iter(self._iter_layer_registration_regions(layer_name))
)
end = max(end, kv_caches[layer_name].data_ptr() - ptr + region_len)
reg_nbytes = ((slack + end + page_size - 1) // page_size) * page_size
if reg_nbytes > nbytes:
raise ValueError(
f"shared KV backing too small for page-aligned MR: "
f"need {reg_nbytes}, have {nbytes}"
)
reg = torch.as_strided(base, (reg_nbytes,), (1,), 0 if slack == 0 else -slack)
mr_ptr = reg.data_ptr()
return reg, {
name: cache.data_ptr() - mr_ptr for name, cache in kv_caches.items()
}
def _remote_layer_mr_offset(
self,
layer_name: str,
remote_moriio_meta: MoRIIOAgentMetadata,
remote_engine_id: EngineId | None,
) -> int:
if not self.kv_layer_mr_offset or not remote_engine_id:
return 0
idx = self.layer_base_addr_index[layer_name]
metas = self.layer_name_to_remote_kv_cache_metadata.get(
remote_engine_id, {}
).get(layer_name)
if not metas or idx >= len(remote_moriio_meta.kv_caches_base_addr):
return 0
mr_base = self.moriio_wrapper.get_unpack_memory_metadata(metas[0]).data
return remote_moriio_meta.kv_caches_base_addr[idx] - mr_base
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
"""Register the KV Cache data in moriio."""
@@ -1754,8 +1800,11 @@ class MoRIIOConnectorWorker:
self.dst_num_blocks[self.engine_id] = self.num_blocks
kv_caches_base_addr = []
caches_data = []
self.layer_base_addr_index = {}
base_addr_idx = 0
for layer_name in kv_caches:
self.layer_base_addr_index[layer_name] = base_addr_idx
geometry = self._get_layer_transfer_geometry(layer_name)
if geometry.block_size != self.block_size:
raise ValueError(
@@ -1767,12 +1816,21 @@ class MoRIIOConnectorWorker:
base_addr = cache.data_ptr()
caches_data.append((base_addr, region_len, cache.device.index, ""))
kv_caches_base_addr.append(base_addr)
base_addr_idx += 1
shared_backing = len({id(t.untyped_storage()) for t in kv_caches.values()}) == 1
shared_mr = None
if shared_backing:
reg_tensor, self.kv_layer_mr_offset = self._build_shared_kv_mr(kv_caches)
shared_mr = self.moriio_wrapper.register_local_tensor(reg_tensor)
for layer_name, kv_cache in kv_caches.items():
if layer_name not in self.layer_name_to_local_kv_cache_metadata:
self.layer_name_to_local_kv_cache_metadata[layer_name] = []
moriio_mem_metadata = self.moriio_wrapper.register_local_tensor(kv_cache)
moriio_mem_metadata = shared_mr or (
self.moriio_wrapper.register_local_tensor(kv_cache)
)
self.layer_name_to_local_kv_cache_metadata[layer_name].append(
moriio_mem_metadata
)
@@ -2488,6 +2546,7 @@ class MoRIIOConnectorWorker:
remote_block_ids: list[int],
remote_moriio_meta: MoRIIOAgentMetadata,
remote_tp_size: int | None = None,
remote_engine_id: EngineId | None = None,
) -> tuple[list[int], list[int], list[int]]:
"""Compute transfer offsets for block data.
@@ -2509,7 +2568,7 @@ class MoRIIOConnectorWorker:
total_num_kv_heads=self.model_config.get_total_num_kv_heads(),
is_mla=self._is_mla_cache_layer(layer_name),
)
return compute_block_transfer_offsets(
local, remote, sizes = compute_block_transfer_offsets(
layer_name=layer_name,
kv_cache=self.kv_caches[layer_name],
layer_to_spec=self.layer_to_spec,
@@ -2520,6 +2579,14 @@ class MoRIIOConnectorWorker:
local, remote, sizes, assume_sorted=False
),
)
if self.kv_layer_mr_offset:
local_off = self.kv_layer_mr_offset[layer_name]
remote_off = self._remote_layer_mr_offset(
layer_name, remote_moriio_meta, remote_engine_id
)
local = [x + local_off for x in local]
remote = [x + remote_off for x in remote]
return local, remote, sizes
@staticmethod
def _is_sq_full_status(status) -> bool:
@@ -2592,6 +2659,7 @@ class MoRIIOConnectorWorker:
remote_block_ids,
remote_moriio_meta,
remote_tp_size=remote_tp_size,
remote_engine_id=remote_dp_engine_id,
)
# TODO : apply multi-session batch-read when moriio support it
#
@@ -377,16 +377,17 @@ class MoRIIOWriter:
The transfer plan
"""
layer_cache = self.worker.kv_caches[task.layer_name]
geometry_key = _get_write_geometry_key(layer_cache)
offsets = request_info.transfer_offsets.get(geometry_key)
key = (task.layer_name, *_get_write_geometry_key(layer_cache))
offsets = request_info.transfer_offsets.get(key)
if offsets is None:
offsets = self.worker._compute_block_transfer_offsets(
task.layer_name,
task.local_block_ids,
request_info.block_ids,
remote_moriio_meta,
remote_engine_id=task.dst_engine_id,
)
request_info.transfer_offsets[geometry_key] = offsets
request_info.transfer_offsets[key] = offsets
# Get session index
layer_names = list(self.worker.layer_name_to_local_kv_cache_metadata.keys())
+7 -1
View File
@@ -392,7 +392,13 @@ def allocate_kv_cache(
sizes = {tensor.size for tensor in kv_cache_config.kv_cache_tensors}
assert len(sizes) == 1, "KV cache tensors must share one backing allocation."
buf = torch.zeros(sizes.pop(), dtype=torch.int8, device=device)
raw_size = sizes.pop()
page_size = 4096
buf = torch.zeros(
((raw_size + page_size - 1) // page_size) * page_size,
dtype=torch.int8,
device=device,
)
kv_caches: dict[str, torch.Tensor] = {}
for tensor in kv_cache_config.kv_cache_tensors: