[EPLB] Add nixl-based eplb communicator (#36276)
Signed-off-by: ilmarkov <markovilya197@gmail.com> Signed-off-by: Markov Ilya <markovilya19@gmail.com>
This commit is contained in:
@@ -15,9 +15,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import (
|
||||
nixlXferTelemetry,
|
||||
)
|
||||
from vllm.distributed.nixl_utils import nixlXferTelemetry
|
||||
from vllm.v1.metrics.utils import create_metric_per_engine
|
||||
|
||||
|
||||
|
||||
@@ -3,60 +3,14 @@
|
||||
"""Shared constants, lazy imports and helpers for the NIXL connector."""
|
||||
|
||||
import contextlib
|
||||
import os
|
||||
import sys
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
import zmq
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import make_zmq_socket
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
# Lazy import nixl_wrapper to avoid loading nixl_bindings if nixl is not used
|
||||
try:
|
||||
if "UCX_RCACHE_MAX_UNRELEASED" not in os.environ:
|
||||
# avoid a memory leak in UCX when using NIXL on some models
|
||||
# see: https://github.com/vllm-project/vllm/issues/24264
|
||||
if "nixl" in sys.modules or "rixl" in sys.modules:
|
||||
logger.warning(
|
||||
"NIXL was already imported, we can't reset UCX_RCACHE_MAX_UNRELEASED. "
|
||||
"Please set it to '1024' manually."
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
"Setting UCX_RCACHE_MAX_UNRELEASED to '1024' to avoid a rare "
|
||||
"memory leak in UCX when using NIXL."
|
||||
)
|
||||
os.environ["UCX_RCACHE_MAX_UNRELEASED"] = "1024"
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
from nixl._api import nixl_agent as NixlWrapper
|
||||
from nixl._bindings import nixlXferTelemetry
|
||||
else:
|
||||
from rixl._api import nixl_agent as NixlWrapper
|
||||
from rixl._bindings import nixlXferTelemetry
|
||||
|
||||
logger.info("NIXL is available")
|
||||
except ImportError:
|
||||
logger.warning("NIXL is not available")
|
||||
NixlWrapper = None
|
||||
nixlXferTelemetry = None
|
||||
|
||||
|
||||
try:
|
||||
if not current_platform.is_rocm():
|
||||
from nixl._api import nixl_agent_config
|
||||
else:
|
||||
from rixl._api import nixl_agent_config
|
||||
except ImportError:
|
||||
nixl_agent_config = None
|
||||
logger.warning("NIXL agent config is not available")
|
||||
|
||||
# Supported platforms and types of kv transfer buffer.
|
||||
# {device: tuple of supported kv buffer types}
|
||||
_NIXL_SUPPORTED_DEVICE = {
|
||||
|
||||
@@ -45,8 +45,6 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import (
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import (
|
||||
_NIXL_SUPPORTED_DEVICE,
|
||||
NixlWrapper,
|
||||
nixl_agent_config,
|
||||
zmq_ctx,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import (
|
||||
@@ -54,6 +52,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import
|
||||
compute_mamba_phys_ratio,
|
||||
derive_mamba_conv_split,
|
||||
)
|
||||
from vllm.distributed.nixl_utils import NixlWrapper, nixl_agent_config
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
|
||||
Reference in New Issue
Block a user