[Refactor] Nixl util using lazy init (#41392)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -4,7 +4,7 @@
|
||||
|
||||
import copy
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
@@ -15,9 +15,11 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
from vllm.distributed.nixl_utils import nixlXferTelemetry
|
||||
from vllm.v1.metrics.utils import create_metric_per_engine
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.nixl_utils import nixlXferTelemetry
|
||||
|
||||
|
||||
@dataclass
|
||||
class NixlKVConnectorStats(KVConnectorStats):
|
||||
@@ -40,7 +42,7 @@ class NixlKVConnectorStats(KVConnectorStats):
|
||||
"num_kv_expired_reqs": [],
|
||||
}
|
||||
|
||||
def record_transfer(self, res: nixlXferTelemetry):
|
||||
def record_transfer(self, res: "nixlXferTelemetry"):
|
||||
# Keep metrics units consistent with rest of the code: time us->s
|
||||
self.data["transfer_duration"].append(res.xferDuration / 1e6)
|
||||
self.data["post_duration"].append(res.postDuration / 1e6)
|
||||
|
||||
@@ -198,7 +198,8 @@ class NixlConnectorWorker:
|
||||
engine_id: str,
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
):
|
||||
if NixlWrapper is None:
|
||||
nixl_wrapper_cls = NixlWrapper
|
||||
if nixl_wrapper_cls is None:
|
||||
logger.error("NIXL is not available")
|
||||
raise RuntimeError("NIXL is not available")
|
||||
logger.info("Initializing NIXL wrapper")
|
||||
@@ -284,7 +285,7 @@ class NixlConnectorWorker:
|
||||
else nixl_agent_config(num_threads=num_threads, capture_telemetry=True)
|
||||
)
|
||||
|
||||
self.nixl_wrapper = NixlWrapper(str(uuid.uuid4()), config)
|
||||
self.nixl_wrapper = nixl_wrapper_cls(str(uuid.uuid4()), config)
|
||||
# Map of engine_id -> {rank0: agent_name0, rank1: agent_name1..}.
|
||||
self._remote_agents: dict[EngineId, dict[int, str]] = defaultdict(dict)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user