[Bugfix][EC Connector] Release shared memory on NIXL setup failure (#59875)
Signed-off-by: DGlan <162426117+DGlan@users.noreply.github.com>
This commit is contained in:
@@ -1,10 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
|
||||
import vllm.distributed.nixl_utils as nixl_utils
|
||||
from tests.v1.ec_connector.unit.utils import create_ec_vllm_config
|
||||
from vllm.config.ec_transfer import ECTransferConfig
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
|
||||
@@ -57,6 +59,72 @@ def test_string_false_in_extra_config_leaves_nixl_off(monkeypatch):
|
||||
s.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_missing_nixl_releases_shared_region(monkeypatch):
|
||||
"""A failed constructor leaves no scheduler for the caller to shut down."""
|
||||
# Import before patching to avoid caching the missing-NIXL values.
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.data import nixl # noqa: F401
|
||||
|
||||
region = _region()
|
||||
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: region)
|
||||
monkeypatch.setattr(nixl_utils, "NixlWrapper", None)
|
||||
monkeypatch.setattr(nixl_utils, "nixl_agent_config", None)
|
||||
cfg = create_ec_vllm_config(ec_role="ec_both")
|
||||
cfg.ec_transfer_config.ec_connector_extra_config["ec_enable_nixl"] = True
|
||||
|
||||
try:
|
||||
with pytest.raises(RuntimeError, match="ec_enable_nixl requires NIXL"):
|
||||
ECCPUScheduler(cfg)
|
||||
|
||||
assert not Path(region._mmap_path).exists()
|
||||
assert region._fd is None
|
||||
assert region._mmap_obj is None
|
||||
finally:
|
||||
region.cleanup()
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_control_transport_failure_releases_nixl_and_shared_region(monkeypatch):
|
||||
import vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq as zmq_mod
|
||||
import vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl as data_mod
|
||||
|
||||
class RegisteredData:
|
||||
registered = True
|
||||
|
||||
def deregister(self):
|
||||
self.registered = False
|
||||
|
||||
data = RegisteredData()
|
||||
error = RuntimeError("control transport failed")
|
||||
|
||||
def fail_control_transport():
|
||||
raise error
|
||||
|
||||
region = _region()
|
||||
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: region)
|
||||
monkeypatch.setattr(nixl_utils, "NixlWrapper", object())
|
||||
monkeypatch.setattr(nixl_utils, "nixl_agent_config", object())
|
||||
monkeypatch.setattr(data_mod, "NixlDataTransport", lambda **kwargs: data)
|
||||
monkeypatch.setattr(zmq_mod, "ZmqClientTransport", fail_control_transport)
|
||||
cfg = create_ec_vllm_config(ec_role="ec_consumer")
|
||||
cfg.model_config.hf_config = None
|
||||
cfg.model_config.get_inputs_embeds_size.return_value = 8
|
||||
cfg.model_config.model = "test-model"
|
||||
cfg.ec_transfer_config.ec_connector_extra_config["ec_enable_nixl"] = True
|
||||
|
||||
try:
|
||||
with pytest.raises(RuntimeError, match="control transport failed") as exc:
|
||||
ECCPUScheduler(cfg)
|
||||
|
||||
assert exc.value is error
|
||||
assert not data.registered
|
||||
assert not Path(region._mmap_path).exists()
|
||||
assert region._fd is None
|
||||
assert region._mmap_obj is None
|
||||
finally:
|
||||
region.cleanup()
|
||||
|
||||
|
||||
@pytest.mark.skipif(NixlWrapper is None, reason="Requires NIXL package")
|
||||
def test_gate_on_wires_data_transport_and_producer_session(monkeypatch):
|
||||
# Port 0 lets the OS pick an ephemeral port so the real ZMQ ROUTER bind
|
||||
|
||||
@@ -115,7 +115,11 @@ class ECCPUScheduler:
|
||||
self._element_size: int = 0
|
||||
self._ack_timeout_s: float = 0.0
|
||||
if self._nixl_enabled:
|
||||
self._setup_nixl(vllm_config)
|
||||
try:
|
||||
self._setup_nixl(vllm_config)
|
||||
except Exception:
|
||||
self.shutdown()
|
||||
raise
|
||||
|
||||
def _setup_nixl(self, vllm_config: "VllmConfig") -> None:
|
||||
# Lazy imports keep nixl/zmq off the gate-off path.
|
||||
@@ -165,35 +169,26 @@ class ECCPUScheduler:
|
||||
self._peer_host = envs.VLLM_EC_SIDE_CHANNEL_HOST
|
||||
self._peer_port = envs.VLLM_EC_SIDE_CHANNEL_PORT
|
||||
|
||||
# Registering the region with NIXL and binding the control sockets are
|
||||
# the first side effects here. __init__ propagates a failure, so the
|
||||
# caller never receives a scheduler it could shut down: unwind through
|
||||
# the same teardown shutdown() uses.
|
||||
try:
|
||||
self._data = NixlDataTransport(
|
||||
agent_name=engine_id,
|
||||
base_ptr=self._region.blocks.data_ptr(),
|
||||
num_blocks=self._region.num_blocks,
|
||||
block_size_bytes=self._region.block_size_bytes,
|
||||
total_size_bytes=self._region.num_blocks
|
||||
* self._region.block_size_bytes,
|
||||
self._data = NixlDataTransport(
|
||||
agent_name=engine_id,
|
||||
base_ptr=self._region.blocks.data_ptr(),
|
||||
num_blocks=self._region.num_blocks,
|
||||
block_size_bytes=self._region.block_size_bytes,
|
||||
total_size_bytes=self._region.num_blocks * self._region.block_size_bytes,
|
||||
)
|
||||
if self._is_producer:
|
||||
assert self._peer_host is not None
|
||||
assert self._peer_port is not None
|
||||
self._producer_session = ProducerSession(
|
||||
transport=ZmqServerTransport(
|
||||
host=self._peer_host, port=self._peer_port
|
||||
),
|
||||
data=self._data,
|
||||
cache=self._cache,
|
||||
compat_hash=self._compat_hash,
|
||||
)
|
||||
if self._is_producer:
|
||||
assert self._peer_host is not None
|
||||
assert self._peer_port is not None
|
||||
self._producer_session = ProducerSession(
|
||||
transport=ZmqServerTransport(
|
||||
host=self._peer_host, port=self._peer_port
|
||||
),
|
||||
data=self._data,
|
||||
cache=self._cache,
|
||||
compat_hash=self._compat_hash,
|
||||
)
|
||||
if self._is_consumer:
|
||||
self._transport = ZmqClientTransport()
|
||||
except Exception:
|
||||
self._teardown_nixl()
|
||||
raise
|
||||
if self._is_consumer:
|
||||
self._transport = ZmqClientTransport()
|
||||
|
||||
def has_cache_item(self, identifier: str) -> bool:
|
||||
if not self._is_consumer:
|
||||
|
||||
Reference in New Issue
Block a user