[Fast Start] Use GPU uuid as socket folder identifier (#56669)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-09-14 06:28:13 +00:00
committed by GitHub
parent cf1584f373
commit dbf49dad11
4 changed files with 30 additions and 69 deletions
@@ -60,6 +60,7 @@ from vllm.distributed import (
ensure_model_parallel_initialized,
init_distributed_environment,
)
from vllm.engine.arg_utils import EngineArgs
from vllm.logger import init_logger
from vllm.model_executor.model_loader import get_model_loader
from vllm.model_executor.model_loader.utils import process_weights_after_loading
@@ -70,13 +71,15 @@ from vllm.model_executor.model_loader.weight_cache.protocol import (
check_ipc_platform_support,
check_ipc_quant_support,
ensure_private_socket_dir,
get_physical_device_id,
get_current_device_uuid,
get_socket_path,
recv_msg,
send_msg,
verify_peer_is_owner,
)
from vllm.platforms import current_platform
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.network_utils import get_distributed_init_method, get_open_port
from vllm.utils.torch_utils import set_default_torch_dtype
logger = init_logger("vllm.model_executor.model_loader.weight_cache.daemon")
@@ -209,7 +212,7 @@ class WeightCacheDaemon:
ready_callback: Invoked once the socket is bound and listening,
so the launcher can report overall readiness.
"""
socket_path = self._socket_path()
socket_path = self._socket_path
ensure_private_socket_dir(
os.path.dirname(socket_path), strict_perms=self.socket_dir is None
)
@@ -268,12 +271,9 @@ class WeightCacheDaemon:
) from e
return lock_fd
@property
def _socket_path(self) -> str:
device_index = torch.accelerator.current_device_index()
gpu_id = get_physical_device_id(device_index)
if gpu_id is None:
gpu_id = device_index
return get_socket_path(gpu_id, self.socket_dir)
return get_socket_path(get_current_device_uuid(), self.socket_dir)
def _handle_connection(self, conn: socket.socket) -> None:
request = recv_msg(conn)
@@ -304,7 +304,7 @@ class WeightCacheDaemon:
"status": "ok",
"entries": self.entries,
"aliases": self.aliases,
"gpu_uuid": self._gpu_uuid(),
"gpu_uuid": get_current_device_uuid(),
},
)
logger.info_once(
@@ -322,12 +322,6 @@ class WeightCacheDaemon:
logger.info("Weight cache daemon rank %d released cached weights", self.tp_rank)
send_msg(conn, {"status": "ok"})
def _gpu_uuid(self) -> str:
props = torch.cuda.get_device_properties(
torch.accelerator.current_device_index()
)
return str(props.uuid)
def _run_daemon(
tp_rank: int,
@@ -359,10 +353,6 @@ def _reject_unsupported_parallelism(parallel_config: ParallelConfig) -> None:
def main() -> None:
from vllm.engine.arg_utils import EngineArgs
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.network_utils import get_distributed_init_method, get_open_port
parser = FlexibleArgumentParser(
description="Launch weight cache daemons (one per TP rank)."
)
@@ -12,6 +12,10 @@ import torch.nn as nn
from vllm.config import ModelConfig, VllmConfig
from vllm.config.load import LoadConfig
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
from vllm.model_executor.model_loader.base_loader import BaseModelLoader
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
@@ -27,7 +31,7 @@ from vllm.model_executor.model_loader.weight_cache.protocol import (
WeightCacheUnavailableError,
check_ipc_platform_support,
check_ipc_quant_support,
get_physical_device_id,
get_current_device_uuid,
get_socket_path,
recv_msg,
send_msg,
@@ -56,7 +60,7 @@ class IpcModelLoader(BaseModelLoader):
Extra config keys (via --model-loader-extra-config):
- socket_path: explicit daemon socket path. Defaults to a per-GPU path
derived from the physical GPU id.
derived from the physical GPU uuid.
- socket_dir: directory containing the daemon sockets.
- mode: "zero_copy" (default) or "copy".
- fallback: fall back to disk loading when the daemon is unavailable or
@@ -261,11 +265,6 @@ class IpcModelLoader(BaseModelLoader):
def _fetch_entries(
self, model_config: ModelConfig
) -> tuple[dict[str, TensorEntry], dict[str, str]]:
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
cache_config = WeightCacheKey.from_model_config(
model_config,
tp_size=get_tensor_model_parallel_world_size(),
@@ -317,22 +316,12 @@ class IpcModelLoader(BaseModelLoader):
def _resolve_socket_path(self) -> str:
if self.socket_path is not None:
return self.socket_path
device_index = torch.accelerator.current_device_index()
gpu_id = get_physical_device_id(device_index)
if gpu_id is None:
raise WeightCacheUnavailableError(
"Cannot infer the physical GPU id from CUDA_VISIBLE_DEVICES; "
"pass socket_path via --model-loader-extra-config"
)
return get_socket_path(gpu_id, self.socket_dir)
return get_socket_path(get_current_device_uuid(), self.socket_dir)
def _check_gpu_uuid(self, daemon_uuid: str | None) -> None:
if daemon_uuid is None:
return
props = torch.cuda.get_device_properties(
torch.accelerator.current_device_index()
)
local_uuid = str(props.uuid)
local_uuid = get_current_device_uuid()
if daemon_uuid != local_uuid:
raise CacheConfigMismatchError(
f"Daemon GPU {daemon_uuid} != engine GPU {local_uuid}; "
@@ -23,14 +23,19 @@ from dataclasses import dataclass, fields
from typing import Any
import torch
from torch.multiprocessing.reductions import rebuild_cuda_tensor, reduce_tensor
from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
import vllm.version
from vllm.config import ModelConfig
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
from vllm.model_executor.model_loader.weight_utils import (
filter_duplicate_safetensors_files,
)
from vllm.platforms import current_platform
from vllm.utils.hashing import safe_hash
SOCKET_NAME_TEMPLATE = "vllm_weight_cache_gpu{gpu_id}.sock"
SOCKET_NAME_TEMPLATE = "vllm_weight_cache_{gpu_uuid}.sock"
SOCKET_DIR_TEMPLATE = "vllm_weight_cache_{uid}"
_LEN_STRUCT = struct.Struct("!Q")
@@ -102,20 +107,9 @@ def check_ipc_quant_support(model: torch.nn.Module) -> None:
)
def get_physical_device_id(device_index: int) -> int | None:
"""Map a local CUDA device index to the physical GPU id.
Returns None if CUDA_VISIBLE_DEVICES contains non-integer entries
(e.g. GPU UUIDs), in which case an explicit socket path is required.
"""
visible = os.environ.get("CUDA_VISIBLE_DEVICES")
if not visible:
return device_index
entries = visible.split(",")
try:
return int(entries[device_index])
except (IndexError, ValueError):
return None
def get_current_device_uuid() -> str:
"""UUID of the physical GPU backing the current accelerator device."""
return current_platform.get_device_uuid(torch.accelerator.current_device_index())
def get_socket_dir(socket_dir: str | None = None) -> str:
@@ -132,9 +126,9 @@ def get_socket_dir(socket_dir: str | None = None) -> str:
)
def get_socket_path(gpu_id: int, socket_dir: str | None = None) -> str:
def get_socket_path(gpu_uuid: str, socket_dir: str | None = None) -> str:
directory = get_socket_dir(socket_dir)
return os.path.join(directory, SOCKET_NAME_TEMPLATE.format(gpu_id=gpu_id))
return os.path.join(directory, SOCKET_NAME_TEMPLATE.format(gpu_uuid=gpu_uuid))
def ensure_private_socket_dir(directory: str, strict_perms: bool = True) -> None:
@@ -245,12 +239,6 @@ def hash_checkpoint(model: str) -> str | None:
"""
if not os.path.isdir(model):
return None
from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
from vllm.model_executor.model_loader.weight_utils import (
filter_duplicate_safetensors_files,
)
files = glob.glob(os.path.join(model, "*.safetensors"))
if os.path.isfile(os.path.join(model, SAFE_WEIGHTS_INDEX_NAME)):
files = filter_duplicate_safetensors_files(
@@ -337,8 +325,6 @@ class TensorEntry:
@classmethod
def from_tensor(cls, tensor: torch.Tensor, kind: str) -> "TensorEntry":
from torch.multiprocessing.reductions import reduce_tensor
tensor = tensor.detach()
if tensor.is_cuda:
_, ipc_args = reduce_tensor(tensor)
@@ -349,8 +335,6 @@ class TensorEntry:
if self.ipc_args is None:
assert self.cpu_tensor is not None
return self.cpu_tensor
from torch.multiprocessing.reductions import rebuild_cuda_tensor
args = list(self.ipc_args)
# Index 6 of the args from reduce_tensor is the device index. It must
# be retargeted to the local index since the daemon and the engine may