diff --git a/tests/distributed/test_sharded_rdt_producer.py b/tests/distributed/test_sharded_rdt_producer.py index 62fc638762..5cfab1957d 100644 --- a/tests/distributed/test_sharded_rdt_producer.py +++ b/tests/distributed/test_sharded_rdt_producer.py @@ -20,17 +20,23 @@ covered by the GPU tests in `test_sharded_rdt_trainer.py`. import contextlib import gc +import os +import sys import threading import time from concurrent.futures import ThreadPoolExecutor from concurrent.futures import TimeoutError as FuturesTimeout +from types import ModuleType, SimpleNamespace +from unittest.mock import Mock import pytest import torch import vllm.distributed.weight_transfer.sharded_rdt_trainer as trainer_mod +from vllm.distributed.nixl_utils import alias_nixl_for_ray from vllm.distributed.weight_transfer.sharded_rdt_common import ( buffer_alloc_bytes, + register_nixl_memory, ) from vllm.distributed.weight_transfer.sharded_rdt_trainer import ( DEFAULT_GATHER_LOOKAHEAD, @@ -45,6 +51,114 @@ GROUP_A = ("model.layers.0.w", "model.layers.0.b") GROUP_B = ("model.layers.1.w",) +@pytest.fixture +def ray_nixl_transport(monkeypatch): + import ray + from ray.experimental.rdt import util + from ray.experimental.rdt.nixl_tensor_transport import NixlTensorTransport + + from vllm.distributed import nixl_utils + from vllm.platforms import current_platform + + transport = NixlTensorTransport() + package = ModuleType("nixl_rocm") + api = ModuleType("nixl_rocm._api") + api.__dict__.update(nixl_agent=Mock(), nixl_agent_config=Mock()) + package.__dict__["_api"] = api + modules = {package.__name__: package, api.__name__: api} + imported_rcache_limits = [] + + def import_nixl(name): + imported_rcache_limits.append(os.environ.get("UCX_RCACHE_MAX_UNRELEASED")) + monkeypatch.setitem(sys.modules, name, modules[name]) + return modules[name] + + nixl_names = ("nixl", "nixl._api", *modules) + for name in nixl_names: + monkeypatch.delitem(sys.modules, name, raising=False) + monkeypatch.delenv("UCX_RCACHE_MAX_UNRELEASED", raising=False) + importer = Mock(side_effect=import_nixl) + monkeypatch.setattr( + nixl_utils, "importlib", SimpleNamespace(import_module=importer) + ) + monkeypatch.setattr(nixl_utils, "is_nixl_available", lambda: True) + monkeypatch.setattr(util, "transport_managers", {"NIXL": transport}) + monkeypatch.setattr(current_platform, "is_rocm", lambda: True) + monkeypatch.setattr( + ray, + "get_runtime_context", + lambda: SimpleNamespace(get_actor_id=lambda: "rdt-actor"), + ) + try: + yield transport, api, importer, imported_rcache_limits + finally: + # The helper inserts aliases directly; remove them before monkeypatch + # restores any modules and environment values present before this test. + for name in nixl_names: + sys.modules.pop(name, None) + os.environ.pop("UCX_RCACHE_MAX_UNRELEASED", None) + + +def test_ray_registers_memory_through_rocm_alias(ray_nixl_transport): + """Ray's registration must reach the ROCm agent through the alias.""" + _, api, importer, _ = ray_nixl_transport + register_nixl_memory(torch.empty(4, dtype=torch.uint8)) + api.nixl_agent.return_value.register_memory.assert_called() + assert sys.modules["nixl"] is sys.modules["nixl_rocm"] + assert sys.modules["nixl._api"] is api + + importer.reset_mock() + alias_nixl_for_ray() + importer.assert_not_called() + + +@pytest.mark.parametrize("rcache_limit", [None, "4096"]) +def test_ray_nixl_sets_ucx_limit_before_import( + monkeypatch, ray_nixl_transport, rcache_limit +): + """UCX must see the default or the user's override during the first import.""" + _, _, _, imported_rcache_limits = ray_nixl_transport + if rcache_limit is not None: + monkeypatch.setenv("UCX_RCACHE_MAX_UNRELEASED", rcache_limit) + alias_nixl_for_ray() + assert imported_rcache_limits == [rcache_limit or "1024"] * 2 + + +@pytest.mark.parametrize("existing_api", [False, True]) +def test_ray_nixl_preserves_non_rocm_or_existing_imports( + monkeypatch, ray_nixl_transport, existing_api +): + from vllm.platforms import current_platform + + _, _, importer, _ = ray_nixl_transport + if existing_api: + api = ModuleType("nixl._api") + monkeypatch.setitem(sys.modules, "nixl._api", api) + else: + monkeypatch.setattr(current_platform, "is_rocm", lambda: False) + alias_nixl_for_ray() + importer.assert_not_called() + assert "UCX_RCACHE_MAX_UNRELEASED" not in os.environ + if existing_api: + assert sys.modules["nixl._api"] is api + else: + assert "nixl._api" not in sys.modules + + +def test_ray_nixl_alias_reports_missing_rocm_package(monkeypatch, ray_nixl_transport): + from vllm.distributed import nixl_utils + + transport, api, importer, _ = ray_nixl_transport + monkeypatch.setattr(nixl_utils, "is_nixl_available", lambda: False) + with pytest.raises(ImportError, match="nixl_rocm"): + register_nixl_memory(torch.empty(1)) + assert transport._nixl_agent is None + api.nixl_agent.assert_not_called() + importer.assert_not_called() + assert "nixl" not in sys.modules + assert "nixl._api" not in sys.modules + + @pytest.fixture def server_factory(monkeypatch): """Build real `_RDTProducerServer` instances without a GPU. diff --git a/vllm/distributed/nixl_utils.py b/vllm/distributed/nixl_utils.py index 3879bee0ce..c063b3ea74 100644 --- a/vllm/distributed/nixl_utils.py +++ b/vllm/distributed/nixl_utils.py @@ -91,9 +91,26 @@ def is_nixl_available() -> bool: return pkg in sys.modules or importlib.util.find_spec(pkg) is not None +def alias_nixl_for_ray() -> None: + """Let Ray's ``nixl._api`` import resolve to the ROCm implementation.""" + if not current_platform.is_rocm() or "nixl._api" in sys.modules: + return + + package_name = _get_nixl_package_name() + if not is_nixl_available(): + raise ImportError(f"Ray's NIXL transport on ROCm requires {package_name}") + + _maybe_set_ucx_rcache_limit() + package = importlib.import_module(package_name) + api = importlib.import_module(f"{package_name}._api") + sys.modules.setdefault("nixl", package) + sys.modules.setdefault("nixl._api", api) + + __all__ = [ "NixlWrapper", "nixl_agent_config", "nixlXferTelemetry", "is_nixl_available", + "alias_nixl_for_ray", ] diff --git a/vllm/distributed/weight_transfer/sharded_rdt_common.py b/vllm/distributed/weight_transfer/sharded_rdt_common.py index a5a7ed9233..70110517d3 100644 --- a/vllm/distributed/weight_transfer/sharded_rdt_common.py +++ b/vllm/distributed/weight_transfer/sharded_rdt_common.py @@ -77,6 +77,16 @@ def check_ray_rdt_version() -> None: ) +def register_nixl_memory(tensor: torch.Tensor) -> None: + """Register a tensor with Ray using the platform's NIXL package.""" + from vllm.distributed.nixl_utils import alias_nixl_for_ray + + alias_nixl_for_ray() + from ray.experimental import register_nixl_memory as ray_register_nixl_memory + + ray_register_nixl_memory(tensor) + + def assign_producer_indices( num_producers: int, num_consumers: int, consumer_idx: int ) -> list[int]: diff --git a/vllm/distributed/weight_transfer/sharded_rdt_engine.py b/vllm/distributed/weight_transfer/sharded_rdt_engine.py index 2b8187007b..96dd53f0fd 100644 --- a/vllm/distributed/weight_transfer/sharded_rdt_engine.py +++ b/vllm/distributed/weight_transfer/sharded_rdt_engine.py @@ -66,6 +66,7 @@ from vllm.distributed.weight_transfer.sharded_rdt_common import ( RdtRouter, buffer_alloc_bytes, check_ray_rdt_version, + register_nixl_memory, ) from vllm.distributed.weight_transfer.sharded_rdt_fake import ( BakeSink, @@ -588,8 +589,6 @@ class ShardedRDTWeightTransferEngine( plan = self._cached_plan if plan is None or not plan.chunks: return - from ray.experimental import register_nixl_memory - # (a) consumer receive buffers — one per ring slot, at the largest chunk. max_pack = max(c.pack_bytes for c in plan.chunks) alloc = buffer_alloc_bytes(max_pack, self._buffer_presize) @@ -740,7 +739,7 @@ class ShardedRDTWeightTransferEngine( any time after the metadata push. See the doc's "slot generation handshake". """ - from ray.experimental import register_nixl_memory, set_target_for_ref + from ray.experimental import set_target_for_ref if self._slot_read_done: with self._slot_cv: diff --git a/vllm/distributed/weight_transfer/sharded_rdt_trainer.py b/vllm/distributed/weight_transfer/sharded_rdt_trainer.py index 60d0e60fb0..1be17c4fa0 100644 --- a/vllm/distributed/weight_transfer/sharded_rdt_trainer.py +++ b/vllm/distributed/weight_transfer/sharded_rdt_trainer.py @@ -47,6 +47,7 @@ from vllm.distributed.weight_transfer.sharded_rdt_common import ( ALLOWED_OPS, buffer_alloc_bytes, check_ray_rdt_version, + register_nixl_memory, ) from vllm.logger import init_logger @@ -334,8 +335,6 @@ class _RDTProducerServer: doc). The warmup buffer stays registered so the agent's CUDA-HMEM path stays initialized. """ - from ray.experimental import register_nixl_memory - self._nixl_warmup_buf = torch.zeros(1 << 20, dtype=torch.uint8, device="cuda") with self._reg_lock: register_nixl_memory(self._nixl_warmup_buf) @@ -510,8 +509,6 @@ class _RDTProducerServer: """Allocate + NIXL-register one serve slot: the single allocation seam, so registration cannot be skipped on either of the two paths that make buffers (the init-time reservation and the serve-path backstop).""" - from ray.experimental import register_nixl_memory - t = torch.empty(nbytes, dtype=torch.uint8, device=self._serve_device) with self._reg_lock: register_nixl_memory(t)