[ROCm][Bugfix] Initialize Ray NIXL agents for sharded RDT (#56687)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Andreas Karatzas
2026-09-15 00:34:21 +00:00
committed by GitHub
co-authored by Codex
parent bbbd0a02c9
commit a7576447b8
5 changed files with 144 additions and 7 deletions
@@ -20,17 +20,23 @@ covered by the GPU tests in `test_sharded_rdt_trainer.py`.
import contextlib
import gc
import os
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import TimeoutError as FuturesTimeout
from types import ModuleType, SimpleNamespace
from unittest.mock import Mock
import pytest
import torch
import vllm.distributed.weight_transfer.sharded_rdt_trainer as trainer_mod
from vllm.distributed.nixl_utils import alias_nixl_for_ray
from vllm.distributed.weight_transfer.sharded_rdt_common import (
buffer_alloc_bytes,
register_nixl_memory,
)
from vllm.distributed.weight_transfer.sharded_rdt_trainer import (
DEFAULT_GATHER_LOOKAHEAD,
@@ -45,6 +51,114 @@ GROUP_A = ("model.layers.0.w", "model.layers.0.b")
GROUP_B = ("model.layers.1.w",)
@pytest.fixture
def ray_nixl_transport(monkeypatch):
import ray
from ray.experimental.rdt import util
from ray.experimental.rdt.nixl_tensor_transport import NixlTensorTransport
from vllm.distributed import nixl_utils
from vllm.platforms import current_platform
transport = NixlTensorTransport()
package = ModuleType("nixl_rocm")
api = ModuleType("nixl_rocm._api")
api.__dict__.update(nixl_agent=Mock(), nixl_agent_config=Mock())
package.__dict__["_api"] = api
modules = {package.__name__: package, api.__name__: api}
imported_rcache_limits = []
def import_nixl(name):
imported_rcache_limits.append(os.environ.get("UCX_RCACHE_MAX_UNRELEASED"))
monkeypatch.setitem(sys.modules, name, modules[name])
return modules[name]
nixl_names = ("nixl", "nixl._api", *modules)
for name in nixl_names:
monkeypatch.delitem(sys.modules, name, raising=False)
monkeypatch.delenv("UCX_RCACHE_MAX_UNRELEASED", raising=False)
importer = Mock(side_effect=import_nixl)
monkeypatch.setattr(
nixl_utils, "importlib", SimpleNamespace(import_module=importer)
)
monkeypatch.setattr(nixl_utils, "is_nixl_available", lambda: True)
monkeypatch.setattr(util, "transport_managers", {"NIXL": transport})
monkeypatch.setattr(current_platform, "is_rocm", lambda: True)
monkeypatch.setattr(
ray,
"get_runtime_context",
lambda: SimpleNamespace(get_actor_id=lambda: "rdt-actor"),
)
try:
yield transport, api, importer, imported_rcache_limits
finally:
# The helper inserts aliases directly; remove them before monkeypatch
# restores any modules and environment values present before this test.
for name in nixl_names:
sys.modules.pop(name, None)
os.environ.pop("UCX_RCACHE_MAX_UNRELEASED", None)
def test_ray_registers_memory_through_rocm_alias(ray_nixl_transport):
"""Ray's registration must reach the ROCm agent through the alias."""
_, api, importer, _ = ray_nixl_transport
register_nixl_memory(torch.empty(4, dtype=torch.uint8))
api.nixl_agent.return_value.register_memory.assert_called()
assert sys.modules["nixl"] is sys.modules["nixl_rocm"]
assert sys.modules["nixl._api"] is api
importer.reset_mock()
alias_nixl_for_ray()
importer.assert_not_called()
@pytest.mark.parametrize("rcache_limit", [None, "4096"])
def test_ray_nixl_sets_ucx_limit_before_import(
monkeypatch, ray_nixl_transport, rcache_limit
):
"""UCX must see the default or the user's override during the first import."""
_, _, _, imported_rcache_limits = ray_nixl_transport
if rcache_limit is not None:
monkeypatch.setenv("UCX_RCACHE_MAX_UNRELEASED", rcache_limit)
alias_nixl_for_ray()
assert imported_rcache_limits == [rcache_limit or "1024"] * 2
@pytest.mark.parametrize("existing_api", [False, True])
def test_ray_nixl_preserves_non_rocm_or_existing_imports(
monkeypatch, ray_nixl_transport, existing_api
):
from vllm.platforms import current_platform
_, _, importer, _ = ray_nixl_transport
if existing_api:
api = ModuleType("nixl._api")
monkeypatch.setitem(sys.modules, "nixl._api", api)
else:
monkeypatch.setattr(current_platform, "is_rocm", lambda: False)
alias_nixl_for_ray()
importer.assert_not_called()
assert "UCX_RCACHE_MAX_UNRELEASED" not in os.environ
if existing_api:
assert sys.modules["nixl._api"] is api
else:
assert "nixl._api" not in sys.modules
def test_ray_nixl_alias_reports_missing_rocm_package(monkeypatch, ray_nixl_transport):
from vllm.distributed import nixl_utils
transport, api, importer, _ = ray_nixl_transport
monkeypatch.setattr(nixl_utils, "is_nixl_available", lambda: False)
with pytest.raises(ImportError, match="nixl_rocm"):
register_nixl_memory(torch.empty(1))
assert transport._nixl_agent is None
api.nixl_agent.assert_not_called()
importer.assert_not_called()
assert "nixl" not in sys.modules
assert "nixl._api" not in sys.modules
@pytest.fixture
def server_factory(monkeypatch):
"""Build real `_RDTProducerServer` instances without a GPU.
+17
View File
@@ -91,9 +91,26 @@ def is_nixl_available() -> bool:
return pkg in sys.modules or importlib.util.find_spec(pkg) is not None
def alias_nixl_for_ray() -> None:
"""Let Ray's ``nixl._api`` import resolve to the ROCm implementation."""
if not current_platform.is_rocm() or "nixl._api" in sys.modules:
return
package_name = _get_nixl_package_name()
if not is_nixl_available():
raise ImportError(f"Ray's NIXL transport on ROCm requires {package_name}")
_maybe_set_ucx_rcache_limit()
package = importlib.import_module(package_name)
api = importlib.import_module(f"{package_name}._api")
sys.modules.setdefault("nixl", package)
sys.modules.setdefault("nixl._api", api)
__all__ = [
"NixlWrapper",
"nixl_agent_config",
"nixlXferTelemetry",
"is_nixl_available",
"alias_nixl_for_ray",
]
@@ -77,6 +77,16 @@ def check_ray_rdt_version() -> None:
)
def register_nixl_memory(tensor: torch.Tensor) -> None:
"""Register a tensor with Ray using the platform's NIXL package."""
from vllm.distributed.nixl_utils import alias_nixl_for_ray
alias_nixl_for_ray()
from ray.experimental import register_nixl_memory as ray_register_nixl_memory
ray_register_nixl_memory(tensor)
def assign_producer_indices(
num_producers: int, num_consumers: int, consumer_idx: int
) -> list[int]:
@@ -66,6 +66,7 @@ from vllm.distributed.weight_transfer.sharded_rdt_common import (
RdtRouter,
buffer_alloc_bytes,
check_ray_rdt_version,
register_nixl_memory,
)
from vllm.distributed.weight_transfer.sharded_rdt_fake import (
BakeSink,
@@ -588,8 +589,6 @@ class ShardedRDTWeightTransferEngine(
plan = self._cached_plan
if plan is None or not plan.chunks:
return
from ray.experimental import register_nixl_memory
# (a) consumer receive buffers — one per ring slot, at the largest chunk.
max_pack = max(c.pack_bytes for c in plan.chunks)
alloc = buffer_alloc_bytes(max_pack, self._buffer_presize)
@@ -740,7 +739,7 @@ class ShardedRDTWeightTransferEngine(
any time after the metadata push. See the doc's "slot generation
handshake".
"""
from ray.experimental import register_nixl_memory, set_target_for_ref
from ray.experimental import set_target_for_ref
if self._slot_read_done:
with self._slot_cv:
@@ -47,6 +47,7 @@ from vllm.distributed.weight_transfer.sharded_rdt_common import (
ALLOWED_OPS,
buffer_alloc_bytes,
check_ray_rdt_version,
register_nixl_memory,
)
from vllm.logger import init_logger
@@ -334,8 +335,6 @@ class _RDTProducerServer:
doc). The warmup buffer stays registered so the agent's CUDA-HMEM path
stays initialized.
"""
from ray.experimental import register_nixl_memory
self._nixl_warmup_buf = torch.zeros(1 << 20, dtype=torch.uint8, device="cuda")
with self._reg_lock:
register_nixl_memory(self._nixl_warmup_buf)
@@ -510,8 +509,6 @@ class _RDTProducerServer:
"""Allocate + NIXL-register one serve slot: the single allocation seam,
so registration cannot be skipped on either of the two paths that make
buffers (the init-time reservation and the serve-path backstop)."""
from ray.experimental import register_nixl_memory
t = torch.empty(nbytes, dtype=torch.uint8, device=self._serve_device)
with self._reg_lock:
register_nixl_memory(t)