From dbf49dad11e30ec5d464be74b715a9a9e31c2c5a Mon Sep 17 00:00:00 2001 From: Isotr0py Date: Mon, 14 Sep 2026 14:28:13 +0800 Subject: [PATCH] [Fast Start] Use GPU uuid as socket folder identifier (#56669) Signed-off-by: Isotr0py --- .../model_loader/test_weight_cache.py | 8 ++-- .../model_loader/weight_cache/daemon.py | 26 ++++--------- .../model_loader/weight_cache/ipc_loader.py | 27 ++++--------- .../model_loader/weight_cache/protocol.py | 38 ++++++------------- 4 files changed, 30 insertions(+), 69 deletions(-) diff --git a/tests/model_executor/model_loader/test_weight_cache.py b/tests/model_executor/model_loader/test_weight_cache.py index fa0b6b4183..1195b8b09d 100644 --- a/tests/model_executor/model_loader/test_weight_cache.py +++ b/tests/model_executor/model_loader/test_weight_cache.py @@ -7,6 +7,7 @@ warm restarts (weights mapped from the daemon via CUDA IPC) must both serve identical outputs. """ +import glob import os import shutil import subprocess @@ -21,7 +22,6 @@ import pytest from vllm import SamplingParams from vllm.assets.image import ImageAsset -from vllm.model_executor.model_loader.weight_cache.protocol import get_socket_path from vllm.platforms import current_platform DAEMON_TIMEOUT_S = 600 @@ -79,9 +79,7 @@ class WeightCacheDaemon: # Poll for the socket files rather than a log line: model loading can # pull in JIT compilers that swap the process's stderr and swallow # everything logged afterwards, making log-based readiness flaky. - expected = [ - get_socket_path(gpu_id, self.socket_dir) for gpu_id in range(self.tp_size) - ] + pattern = os.path.join(self.socket_dir, "vllm_weight_cache_*.sock") deadline = time.monotonic() + timeout_s while time.monotonic() < deadline: if self._proc.poll() is not None: @@ -89,7 +87,7 @@ class WeightCacheDaemon: f"Weight cache daemon exited with {self._proc.returncode}:\n" f"{self._logs()}" ) - if all(os.path.exists(path) for path in expected): + if len(glob.glob(pattern)) >= self.tp_size: return time.sleep(1.0) raise TimeoutError( diff --git a/vllm/model_executor/model_loader/weight_cache/daemon.py b/vllm/model_executor/model_loader/weight_cache/daemon.py index 96cedbe1e9..b19fb96272 100644 --- a/vllm/model_executor/model_loader/weight_cache/daemon.py +++ b/vllm/model_executor/model_loader/weight_cache/daemon.py @@ -60,6 +60,7 @@ from vllm.distributed import ( ensure_model_parallel_initialized, init_distributed_environment, ) +from vllm.engine.arg_utils import EngineArgs from vllm.logger import init_logger from vllm.model_executor.model_loader import get_model_loader from vllm.model_executor.model_loader.utils import process_weights_after_loading @@ -70,13 +71,15 @@ from vllm.model_executor.model_loader.weight_cache.protocol import ( check_ipc_platform_support, check_ipc_quant_support, ensure_private_socket_dir, - get_physical_device_id, + get_current_device_uuid, get_socket_path, recv_msg, send_msg, verify_peer_is_owner, ) from vllm.platforms import current_platform +from vllm.utils.argparse_utils import FlexibleArgumentParser +from vllm.utils.network_utils import get_distributed_init_method, get_open_port from vllm.utils.torch_utils import set_default_torch_dtype logger = init_logger("vllm.model_executor.model_loader.weight_cache.daemon") @@ -209,7 +212,7 @@ class WeightCacheDaemon: ready_callback: Invoked once the socket is bound and listening, so the launcher can report overall readiness. """ - socket_path = self._socket_path() + socket_path = self._socket_path ensure_private_socket_dir( os.path.dirname(socket_path), strict_perms=self.socket_dir is None ) @@ -268,12 +271,9 @@ class WeightCacheDaemon: ) from e return lock_fd + @property def _socket_path(self) -> str: - device_index = torch.accelerator.current_device_index() - gpu_id = get_physical_device_id(device_index) - if gpu_id is None: - gpu_id = device_index - return get_socket_path(gpu_id, self.socket_dir) + return get_socket_path(get_current_device_uuid(), self.socket_dir) def _handle_connection(self, conn: socket.socket) -> None: request = recv_msg(conn) @@ -304,7 +304,7 @@ class WeightCacheDaemon: "status": "ok", "entries": self.entries, "aliases": self.aliases, - "gpu_uuid": self._gpu_uuid(), + "gpu_uuid": get_current_device_uuid(), }, ) logger.info_once( @@ -322,12 +322,6 @@ class WeightCacheDaemon: logger.info("Weight cache daemon rank %d released cached weights", self.tp_rank) send_msg(conn, {"status": "ok"}) - def _gpu_uuid(self) -> str: - props = torch.cuda.get_device_properties( - torch.accelerator.current_device_index() - ) - return str(props.uuid) - def _run_daemon( tp_rank: int, @@ -359,10 +353,6 @@ def _reject_unsupported_parallelism(parallel_config: ParallelConfig) -> None: def main() -> None: - from vllm.engine.arg_utils import EngineArgs - from vllm.utils.argparse_utils import FlexibleArgumentParser - from vllm.utils.network_utils import get_distributed_init_method, get_open_port - parser = FlexibleArgumentParser( description="Launch weight cache daemons (one per TP rank)." ) diff --git a/vllm/model_executor/model_loader/weight_cache/ipc_loader.py b/vllm/model_executor/model_loader/weight_cache/ipc_loader.py index be2e5ebae4..6c3fb742cd 100644 --- a/vllm/model_executor/model_loader/weight_cache/ipc_loader.py +++ b/vllm/model_executor/model_loader/weight_cache/ipc_loader.py @@ -12,6 +12,10 @@ import torch.nn as nn from vllm.config import ModelConfig, VllmConfig from vllm.config.load import LoadConfig +from vllm.distributed import ( + get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, +) from vllm.logger import init_logger from vllm.model_executor.model_loader.base_loader import BaseModelLoader from vllm.model_executor.model_loader.default_loader import DefaultModelLoader @@ -27,7 +31,7 @@ from vllm.model_executor.model_loader.weight_cache.protocol import ( WeightCacheUnavailableError, check_ipc_platform_support, check_ipc_quant_support, - get_physical_device_id, + get_current_device_uuid, get_socket_path, recv_msg, send_msg, @@ -56,7 +60,7 @@ class IpcModelLoader(BaseModelLoader): Extra config keys (via --model-loader-extra-config): - socket_path: explicit daemon socket path. Defaults to a per-GPU path - derived from the physical GPU id. + derived from the physical GPU uuid. - socket_dir: directory containing the daemon sockets. - mode: "zero_copy" (default) or "copy". - fallback: fall back to disk loading when the daemon is unavailable or @@ -261,11 +265,6 @@ class IpcModelLoader(BaseModelLoader): def _fetch_entries( self, model_config: ModelConfig ) -> tuple[dict[str, TensorEntry], dict[str, str]]: - from vllm.distributed import ( - get_tensor_model_parallel_rank, - get_tensor_model_parallel_world_size, - ) - cache_config = WeightCacheKey.from_model_config( model_config, tp_size=get_tensor_model_parallel_world_size(), @@ -317,22 +316,12 @@ class IpcModelLoader(BaseModelLoader): def _resolve_socket_path(self) -> str: if self.socket_path is not None: return self.socket_path - device_index = torch.accelerator.current_device_index() - gpu_id = get_physical_device_id(device_index) - if gpu_id is None: - raise WeightCacheUnavailableError( - "Cannot infer the physical GPU id from CUDA_VISIBLE_DEVICES; " - "pass socket_path via --model-loader-extra-config" - ) - return get_socket_path(gpu_id, self.socket_dir) + return get_socket_path(get_current_device_uuid(), self.socket_dir) def _check_gpu_uuid(self, daemon_uuid: str | None) -> None: if daemon_uuid is None: return - props = torch.cuda.get_device_properties( - torch.accelerator.current_device_index() - ) - local_uuid = str(props.uuid) + local_uuid = get_current_device_uuid() if daemon_uuid != local_uuid: raise CacheConfigMismatchError( f"Daemon GPU {daemon_uuid} != engine GPU {local_uuid}; " diff --git a/vllm/model_executor/model_loader/weight_cache/protocol.py b/vllm/model_executor/model_loader/weight_cache/protocol.py index fff1fcca2a..00d3dbfa8b 100644 --- a/vllm/model_executor/model_loader/weight_cache/protocol.py +++ b/vllm/model_executor/model_loader/weight_cache/protocol.py @@ -23,14 +23,19 @@ from dataclasses import dataclass, fields from typing import Any import torch +from torch.multiprocessing.reductions import rebuild_cuda_tensor, reduce_tensor +from transformers.utils import SAFE_WEIGHTS_INDEX_NAME import vllm.version from vllm.config import ModelConfig from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase +from vllm.model_executor.model_loader.weight_utils import ( + filter_duplicate_safetensors_files, +) from vllm.platforms import current_platform from vllm.utils.hashing import safe_hash -SOCKET_NAME_TEMPLATE = "vllm_weight_cache_gpu{gpu_id}.sock" +SOCKET_NAME_TEMPLATE = "vllm_weight_cache_{gpu_uuid}.sock" SOCKET_DIR_TEMPLATE = "vllm_weight_cache_{uid}" _LEN_STRUCT = struct.Struct("!Q") @@ -102,20 +107,9 @@ def check_ipc_quant_support(model: torch.nn.Module) -> None: ) -def get_physical_device_id(device_index: int) -> int | None: - """Map a local CUDA device index to the physical GPU id. - - Returns None if CUDA_VISIBLE_DEVICES contains non-integer entries - (e.g. GPU UUIDs), in which case an explicit socket path is required. - """ - visible = os.environ.get("CUDA_VISIBLE_DEVICES") - if not visible: - return device_index - entries = visible.split(",") - try: - return int(entries[device_index]) - except (IndexError, ValueError): - return None +def get_current_device_uuid() -> str: + """UUID of the physical GPU backing the current accelerator device.""" + return current_platform.get_device_uuid(torch.accelerator.current_device_index()) def get_socket_dir(socket_dir: str | None = None) -> str: @@ -132,9 +126,9 @@ def get_socket_dir(socket_dir: str | None = None) -> str: ) -def get_socket_path(gpu_id: int, socket_dir: str | None = None) -> str: +def get_socket_path(gpu_uuid: str, socket_dir: str | None = None) -> str: directory = get_socket_dir(socket_dir) - return os.path.join(directory, SOCKET_NAME_TEMPLATE.format(gpu_id=gpu_id)) + return os.path.join(directory, SOCKET_NAME_TEMPLATE.format(gpu_uuid=gpu_uuid)) def ensure_private_socket_dir(directory: str, strict_perms: bool = True) -> None: @@ -245,12 +239,6 @@ def hash_checkpoint(model: str) -> str | None: """ if not os.path.isdir(model): return None - from transformers.utils import SAFE_WEIGHTS_INDEX_NAME - - from vllm.model_executor.model_loader.weight_utils import ( - filter_duplicate_safetensors_files, - ) - files = glob.glob(os.path.join(model, "*.safetensors")) if os.path.isfile(os.path.join(model, SAFE_WEIGHTS_INDEX_NAME)): files = filter_duplicate_safetensors_files( @@ -337,8 +325,6 @@ class TensorEntry: @classmethod def from_tensor(cls, tensor: torch.Tensor, kind: str) -> "TensorEntry": - from torch.multiprocessing.reductions import reduce_tensor - tensor = tensor.detach() if tensor.is_cuda: _, ipc_args = reduce_tensor(tensor) @@ -349,8 +335,6 @@ class TensorEntry: if self.ipc_args is None: assert self.cpu_tensor is not None return self.cpu_tensor - from torch.multiprocessing.reductions import rebuild_cuda_tensor - args = list(self.ipc_args) # Index 6 of the args from reduce_tensor is the device index. It must # be retargeted to the local index since the daemon and the engine may