[ROCm][Bugfix] Initialize Ray NIXL agents for sharded RDT (#56687)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Codex
parent
bbbd0a02c9
commit
a7576447b8
@@ -20,17 +20,23 @@ covered by the GPU tests in `test_sharded_rdt_trainer.py`.
|
||||
|
||||
import contextlib
|
||||
import gc
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from concurrent.futures import TimeoutError as FuturesTimeout
|
||||
from types import ModuleType, SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.distributed.weight_transfer.sharded_rdt_trainer as trainer_mod
|
||||
from vllm.distributed.nixl_utils import alias_nixl_for_ray
|
||||
from vllm.distributed.weight_transfer.sharded_rdt_common import (
|
||||
buffer_alloc_bytes,
|
||||
register_nixl_memory,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.sharded_rdt_trainer import (
|
||||
DEFAULT_GATHER_LOOKAHEAD,
|
||||
@@ -45,6 +51,114 @@ GROUP_A = ("model.layers.0.w", "model.layers.0.b")
|
||||
GROUP_B = ("model.layers.1.w",)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ray_nixl_transport(monkeypatch):
|
||||
import ray
|
||||
from ray.experimental.rdt import util
|
||||
from ray.experimental.rdt.nixl_tensor_transport import NixlTensorTransport
|
||||
|
||||
from vllm.distributed import nixl_utils
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
transport = NixlTensorTransport()
|
||||
package = ModuleType("nixl_rocm")
|
||||
api = ModuleType("nixl_rocm._api")
|
||||
api.__dict__.update(nixl_agent=Mock(), nixl_agent_config=Mock())
|
||||
package.__dict__["_api"] = api
|
||||
modules = {package.__name__: package, api.__name__: api}
|
||||
imported_rcache_limits = []
|
||||
|
||||
def import_nixl(name):
|
||||
imported_rcache_limits.append(os.environ.get("UCX_RCACHE_MAX_UNRELEASED"))
|
||||
monkeypatch.setitem(sys.modules, name, modules[name])
|
||||
return modules[name]
|
||||
|
||||
nixl_names = ("nixl", "nixl._api", *modules)
|
||||
for name in nixl_names:
|
||||
monkeypatch.delitem(sys.modules, name, raising=False)
|
||||
monkeypatch.delenv("UCX_RCACHE_MAX_UNRELEASED", raising=False)
|
||||
importer = Mock(side_effect=import_nixl)
|
||||
monkeypatch.setattr(
|
||||
nixl_utils, "importlib", SimpleNamespace(import_module=importer)
|
||||
)
|
||||
monkeypatch.setattr(nixl_utils, "is_nixl_available", lambda: True)
|
||||
monkeypatch.setattr(util, "transport_managers", {"NIXL": transport})
|
||||
monkeypatch.setattr(current_platform, "is_rocm", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
ray,
|
||||
"get_runtime_context",
|
||||
lambda: SimpleNamespace(get_actor_id=lambda: "rdt-actor"),
|
||||
)
|
||||
try:
|
||||
yield transport, api, importer, imported_rcache_limits
|
||||
finally:
|
||||
# The helper inserts aliases directly; remove them before monkeypatch
|
||||
# restores any modules and environment values present before this test.
|
||||
for name in nixl_names:
|
||||
sys.modules.pop(name, None)
|
||||
os.environ.pop("UCX_RCACHE_MAX_UNRELEASED", None)
|
||||
|
||||
|
||||
def test_ray_registers_memory_through_rocm_alias(ray_nixl_transport):
|
||||
"""Ray's registration must reach the ROCm agent through the alias."""
|
||||
_, api, importer, _ = ray_nixl_transport
|
||||
register_nixl_memory(torch.empty(4, dtype=torch.uint8))
|
||||
api.nixl_agent.return_value.register_memory.assert_called()
|
||||
assert sys.modules["nixl"] is sys.modules["nixl_rocm"]
|
||||
assert sys.modules["nixl._api"] is api
|
||||
|
||||
importer.reset_mock()
|
||||
alias_nixl_for_ray()
|
||||
importer.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rcache_limit", [None, "4096"])
|
||||
def test_ray_nixl_sets_ucx_limit_before_import(
|
||||
monkeypatch, ray_nixl_transport, rcache_limit
|
||||
):
|
||||
"""UCX must see the default or the user's override during the first import."""
|
||||
_, _, _, imported_rcache_limits = ray_nixl_transport
|
||||
if rcache_limit is not None:
|
||||
monkeypatch.setenv("UCX_RCACHE_MAX_UNRELEASED", rcache_limit)
|
||||
alias_nixl_for_ray()
|
||||
assert imported_rcache_limits == [rcache_limit or "1024"] * 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("existing_api", [False, True])
|
||||
def test_ray_nixl_preserves_non_rocm_or_existing_imports(
|
||||
monkeypatch, ray_nixl_transport, existing_api
|
||||
):
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
_, _, importer, _ = ray_nixl_transport
|
||||
if existing_api:
|
||||
api = ModuleType("nixl._api")
|
||||
monkeypatch.setitem(sys.modules, "nixl._api", api)
|
||||
else:
|
||||
monkeypatch.setattr(current_platform, "is_rocm", lambda: False)
|
||||
alias_nixl_for_ray()
|
||||
importer.assert_not_called()
|
||||
assert "UCX_RCACHE_MAX_UNRELEASED" not in os.environ
|
||||
if existing_api:
|
||||
assert sys.modules["nixl._api"] is api
|
||||
else:
|
||||
assert "nixl._api" not in sys.modules
|
||||
|
||||
|
||||
def test_ray_nixl_alias_reports_missing_rocm_package(monkeypatch, ray_nixl_transport):
|
||||
from vllm.distributed import nixl_utils
|
||||
|
||||
transport, api, importer, _ = ray_nixl_transport
|
||||
monkeypatch.setattr(nixl_utils, "is_nixl_available", lambda: False)
|
||||
with pytest.raises(ImportError, match="nixl_rocm"):
|
||||
register_nixl_memory(torch.empty(1))
|
||||
assert transport._nixl_agent is None
|
||||
api.nixl_agent.assert_not_called()
|
||||
importer.assert_not_called()
|
||||
assert "nixl" not in sys.modules
|
||||
assert "nixl._api" not in sys.modules
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def server_factory(monkeypatch):
|
||||
"""Build real `_RDTProducerServer` instances without a GPU.
|
||||
|
||||
@@ -91,9 +91,26 @@ def is_nixl_available() -> bool:
|
||||
return pkg in sys.modules or importlib.util.find_spec(pkg) is not None
|
||||
|
||||
|
||||
def alias_nixl_for_ray() -> None:
|
||||
"""Let Ray's ``nixl._api`` import resolve to the ROCm implementation."""
|
||||
if not current_platform.is_rocm() or "nixl._api" in sys.modules:
|
||||
return
|
||||
|
||||
package_name = _get_nixl_package_name()
|
||||
if not is_nixl_available():
|
||||
raise ImportError(f"Ray's NIXL transport on ROCm requires {package_name}")
|
||||
|
||||
_maybe_set_ucx_rcache_limit()
|
||||
package = importlib.import_module(package_name)
|
||||
api = importlib.import_module(f"{package_name}._api")
|
||||
sys.modules.setdefault("nixl", package)
|
||||
sys.modules.setdefault("nixl._api", api)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"NixlWrapper",
|
||||
"nixl_agent_config",
|
||||
"nixlXferTelemetry",
|
||||
"is_nixl_available",
|
||||
"alias_nixl_for_ray",
|
||||
]
|
||||
|
||||
@@ -77,6 +77,16 @@ def check_ray_rdt_version() -> None:
|
||||
)
|
||||
|
||||
|
||||
def register_nixl_memory(tensor: torch.Tensor) -> None:
|
||||
"""Register a tensor with Ray using the platform's NIXL package."""
|
||||
from vllm.distributed.nixl_utils import alias_nixl_for_ray
|
||||
|
||||
alias_nixl_for_ray()
|
||||
from ray.experimental import register_nixl_memory as ray_register_nixl_memory
|
||||
|
||||
ray_register_nixl_memory(tensor)
|
||||
|
||||
|
||||
def assign_producer_indices(
|
||||
num_producers: int, num_consumers: int, consumer_idx: int
|
||||
) -> list[int]:
|
||||
|
||||
@@ -66,6 +66,7 @@ from vllm.distributed.weight_transfer.sharded_rdt_common import (
|
||||
RdtRouter,
|
||||
buffer_alloc_bytes,
|
||||
check_ray_rdt_version,
|
||||
register_nixl_memory,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.sharded_rdt_fake import (
|
||||
BakeSink,
|
||||
@@ -588,8 +589,6 @@ class ShardedRDTWeightTransferEngine(
|
||||
plan = self._cached_plan
|
||||
if plan is None or not plan.chunks:
|
||||
return
|
||||
from ray.experimental import register_nixl_memory
|
||||
|
||||
# (a) consumer receive buffers — one per ring slot, at the largest chunk.
|
||||
max_pack = max(c.pack_bytes for c in plan.chunks)
|
||||
alloc = buffer_alloc_bytes(max_pack, self._buffer_presize)
|
||||
@@ -740,7 +739,7 @@ class ShardedRDTWeightTransferEngine(
|
||||
any time after the metadata push. See the doc's "slot generation
|
||||
handshake".
|
||||
"""
|
||||
from ray.experimental import register_nixl_memory, set_target_for_ref
|
||||
from ray.experimental import set_target_for_ref
|
||||
|
||||
if self._slot_read_done:
|
||||
with self._slot_cv:
|
||||
|
||||
@@ -47,6 +47,7 @@ from vllm.distributed.weight_transfer.sharded_rdt_common import (
|
||||
ALLOWED_OPS,
|
||||
buffer_alloc_bytes,
|
||||
check_ray_rdt_version,
|
||||
register_nixl_memory,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
@@ -334,8 +335,6 @@ class _RDTProducerServer:
|
||||
doc). The warmup buffer stays registered so the agent's CUDA-HMEM path
|
||||
stays initialized.
|
||||
"""
|
||||
from ray.experimental import register_nixl_memory
|
||||
|
||||
self._nixl_warmup_buf = torch.zeros(1 << 20, dtype=torch.uint8, device="cuda")
|
||||
with self._reg_lock:
|
||||
register_nixl_memory(self._nixl_warmup_buf)
|
||||
@@ -510,8 +509,6 @@ class _RDTProducerServer:
|
||||
"""Allocate + NIXL-register one serve slot: the single allocation seam,
|
||||
so registration cannot be skipped on either of the two paths that make
|
||||
buffers (the init-time reservation and the serve-path backstop)."""
|
||||
from ray.experimental import register_nixl_memory
|
||||
|
||||
t = torch.empty(nbytes, dtype=torch.uint8, device=self._serve_device)
|
||||
with self._reg_lock:
|
||||
register_nixl_memory(t)
|
||||
|
||||
Reference in New Issue
Block a user