[Fast Start] Use GPU uuid as socket folder identifier (#56669)
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -7,6 +7,7 @@ warm restarts (weights mapped from the daemon via CUDA IPC) must both serve
|
||||
identical outputs.
|
||||
"""
|
||||
|
||||
import glob
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
@@ -21,7 +22,6 @@ import pytest
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.model_executor.model_loader.weight_cache.protocol import get_socket_path
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
DAEMON_TIMEOUT_S = 600
|
||||
@@ -79,9 +79,7 @@ class WeightCacheDaemon:
|
||||
# Poll for the socket files rather than a log line: model loading can
|
||||
# pull in JIT compilers that swap the process's stderr and swallow
|
||||
# everything logged afterwards, making log-based readiness flaky.
|
||||
expected = [
|
||||
get_socket_path(gpu_id, self.socket_dir) for gpu_id in range(self.tp_size)
|
||||
]
|
||||
pattern = os.path.join(self.socket_dir, "vllm_weight_cache_*.sock")
|
||||
deadline = time.monotonic() + timeout_s
|
||||
while time.monotonic() < deadline:
|
||||
if self._proc.poll() is not None:
|
||||
@@ -89,7 +87,7 @@ class WeightCacheDaemon:
|
||||
f"Weight cache daemon exited with {self._proc.returncode}:\n"
|
||||
f"{self._logs()}"
|
||||
)
|
||||
if all(os.path.exists(path) for path in expected):
|
||||
if len(glob.glob(pattern)) >= self.tp_size:
|
||||
return
|
||||
time.sleep(1.0)
|
||||
raise TimeoutError(
|
||||
|
||||
@@ -60,6 +60,7 @@ from vllm.distributed import (
|
||||
ensure_model_parallel_initialized,
|
||||
init_distributed_environment,
|
||||
)
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.model_loader import get_model_loader
|
||||
from vllm.model_executor.model_loader.utils import process_weights_after_loading
|
||||
@@ -70,13 +71,15 @@ from vllm.model_executor.model_loader.weight_cache.protocol import (
|
||||
check_ipc_platform_support,
|
||||
check_ipc_quant_support,
|
||||
ensure_private_socket_dir,
|
||||
get_physical_device_id,
|
||||
get_current_device_uuid,
|
||||
get_socket_path,
|
||||
recv_msg,
|
||||
send_msg,
|
||||
verify_peer_is_owner,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
from vllm.utils.network_utils import get_distributed_init_method, get_open_port
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
|
||||
logger = init_logger("vllm.model_executor.model_loader.weight_cache.daemon")
|
||||
@@ -209,7 +212,7 @@ class WeightCacheDaemon:
|
||||
ready_callback: Invoked once the socket is bound and listening,
|
||||
so the launcher can report overall readiness.
|
||||
"""
|
||||
socket_path = self._socket_path()
|
||||
socket_path = self._socket_path
|
||||
ensure_private_socket_dir(
|
||||
os.path.dirname(socket_path), strict_perms=self.socket_dir is None
|
||||
)
|
||||
@@ -268,12 +271,9 @@ class WeightCacheDaemon:
|
||||
) from e
|
||||
return lock_fd
|
||||
|
||||
@property
|
||||
def _socket_path(self) -> str:
|
||||
device_index = torch.accelerator.current_device_index()
|
||||
gpu_id = get_physical_device_id(device_index)
|
||||
if gpu_id is None:
|
||||
gpu_id = device_index
|
||||
return get_socket_path(gpu_id, self.socket_dir)
|
||||
return get_socket_path(get_current_device_uuid(), self.socket_dir)
|
||||
|
||||
def _handle_connection(self, conn: socket.socket) -> None:
|
||||
request = recv_msg(conn)
|
||||
@@ -304,7 +304,7 @@ class WeightCacheDaemon:
|
||||
"status": "ok",
|
||||
"entries": self.entries,
|
||||
"aliases": self.aliases,
|
||||
"gpu_uuid": self._gpu_uuid(),
|
||||
"gpu_uuid": get_current_device_uuid(),
|
||||
},
|
||||
)
|
||||
logger.info_once(
|
||||
@@ -322,12 +322,6 @@ class WeightCacheDaemon:
|
||||
logger.info("Weight cache daemon rank %d released cached weights", self.tp_rank)
|
||||
send_msg(conn, {"status": "ok"})
|
||||
|
||||
def _gpu_uuid(self) -> str:
|
||||
props = torch.cuda.get_device_properties(
|
||||
torch.accelerator.current_device_index()
|
||||
)
|
||||
return str(props.uuid)
|
||||
|
||||
|
||||
def _run_daemon(
|
||||
tp_rank: int,
|
||||
@@ -359,10 +353,6 @@ def _reject_unsupported_parallelism(parallel_config: ParallelConfig) -> None:
|
||||
|
||||
|
||||
def main() -> None:
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
from vllm.utils.network_utils import get_distributed_init_method, get_open_port
|
||||
|
||||
parser = FlexibleArgumentParser(
|
||||
description="Launch weight cache daemons (one per TP rank)."
|
||||
)
|
||||
|
||||
@@ -12,6 +12,10 @@ import torch.nn as nn
|
||||
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.config.load import LoadConfig
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.model_loader.base_loader import BaseModelLoader
|
||||
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
|
||||
@@ -27,7 +31,7 @@ from vllm.model_executor.model_loader.weight_cache.protocol import (
|
||||
WeightCacheUnavailableError,
|
||||
check_ipc_platform_support,
|
||||
check_ipc_quant_support,
|
||||
get_physical_device_id,
|
||||
get_current_device_uuid,
|
||||
get_socket_path,
|
||||
recv_msg,
|
||||
send_msg,
|
||||
@@ -56,7 +60,7 @@ class IpcModelLoader(BaseModelLoader):
|
||||
Extra config keys (via --model-loader-extra-config):
|
||||
|
||||
- socket_path: explicit daemon socket path. Defaults to a per-GPU path
|
||||
derived from the physical GPU id.
|
||||
derived from the physical GPU uuid.
|
||||
- socket_dir: directory containing the daemon sockets.
|
||||
- mode: "zero_copy" (default) or "copy".
|
||||
- fallback: fall back to disk loading when the daemon is unavailable or
|
||||
@@ -261,11 +265,6 @@ class IpcModelLoader(BaseModelLoader):
|
||||
def _fetch_entries(
|
||||
self, model_config: ModelConfig
|
||||
) -> tuple[dict[str, TensorEntry], dict[str, str]]:
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
|
||||
cache_config = WeightCacheKey.from_model_config(
|
||||
model_config,
|
||||
tp_size=get_tensor_model_parallel_world_size(),
|
||||
@@ -317,22 +316,12 @@ class IpcModelLoader(BaseModelLoader):
|
||||
def _resolve_socket_path(self) -> str:
|
||||
if self.socket_path is not None:
|
||||
return self.socket_path
|
||||
device_index = torch.accelerator.current_device_index()
|
||||
gpu_id = get_physical_device_id(device_index)
|
||||
if gpu_id is None:
|
||||
raise WeightCacheUnavailableError(
|
||||
"Cannot infer the physical GPU id from CUDA_VISIBLE_DEVICES; "
|
||||
"pass socket_path via --model-loader-extra-config"
|
||||
)
|
||||
return get_socket_path(gpu_id, self.socket_dir)
|
||||
return get_socket_path(get_current_device_uuid(), self.socket_dir)
|
||||
|
||||
def _check_gpu_uuid(self, daemon_uuid: str | None) -> None:
|
||||
if daemon_uuid is None:
|
||||
return
|
||||
props = torch.cuda.get_device_properties(
|
||||
torch.accelerator.current_device_index()
|
||||
)
|
||||
local_uuid = str(props.uuid)
|
||||
local_uuid = get_current_device_uuid()
|
||||
if daemon_uuid != local_uuid:
|
||||
raise CacheConfigMismatchError(
|
||||
f"Daemon GPU {daemon_uuid} != engine GPU {local_uuid}; "
|
||||
|
||||
@@ -23,14 +23,19 @@ from dataclasses import dataclass, fields
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from torch.multiprocessing.reductions import rebuild_cuda_tensor, reduce_tensor
|
||||
from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
|
||||
|
||||
import vllm.version
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
filter_duplicate_safetensors_files,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.hashing import safe_hash
|
||||
|
||||
SOCKET_NAME_TEMPLATE = "vllm_weight_cache_gpu{gpu_id}.sock"
|
||||
SOCKET_NAME_TEMPLATE = "vllm_weight_cache_{gpu_uuid}.sock"
|
||||
SOCKET_DIR_TEMPLATE = "vllm_weight_cache_{uid}"
|
||||
|
||||
_LEN_STRUCT = struct.Struct("!Q")
|
||||
@@ -102,20 +107,9 @@ def check_ipc_quant_support(model: torch.nn.Module) -> None:
|
||||
)
|
||||
|
||||
|
||||
def get_physical_device_id(device_index: int) -> int | None:
|
||||
"""Map a local CUDA device index to the physical GPU id.
|
||||
|
||||
Returns None if CUDA_VISIBLE_DEVICES contains non-integer entries
|
||||
(e.g. GPU UUIDs), in which case an explicit socket path is required.
|
||||
"""
|
||||
visible = os.environ.get("CUDA_VISIBLE_DEVICES")
|
||||
if not visible:
|
||||
return device_index
|
||||
entries = visible.split(",")
|
||||
try:
|
||||
return int(entries[device_index])
|
||||
except (IndexError, ValueError):
|
||||
return None
|
||||
def get_current_device_uuid() -> str:
|
||||
"""UUID of the physical GPU backing the current accelerator device."""
|
||||
return current_platform.get_device_uuid(torch.accelerator.current_device_index())
|
||||
|
||||
|
||||
def get_socket_dir(socket_dir: str | None = None) -> str:
|
||||
@@ -132,9 +126,9 @@ def get_socket_dir(socket_dir: str | None = None) -> str:
|
||||
)
|
||||
|
||||
|
||||
def get_socket_path(gpu_id: int, socket_dir: str | None = None) -> str:
|
||||
def get_socket_path(gpu_uuid: str, socket_dir: str | None = None) -> str:
|
||||
directory = get_socket_dir(socket_dir)
|
||||
return os.path.join(directory, SOCKET_NAME_TEMPLATE.format(gpu_id=gpu_id))
|
||||
return os.path.join(directory, SOCKET_NAME_TEMPLATE.format(gpu_uuid=gpu_uuid))
|
||||
|
||||
|
||||
def ensure_private_socket_dir(directory: str, strict_perms: bool = True) -> None:
|
||||
@@ -245,12 +239,6 @@ def hash_checkpoint(model: str) -> str | None:
|
||||
"""
|
||||
if not os.path.isdir(model):
|
||||
return None
|
||||
from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
|
||||
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
filter_duplicate_safetensors_files,
|
||||
)
|
||||
|
||||
files = glob.glob(os.path.join(model, "*.safetensors"))
|
||||
if os.path.isfile(os.path.join(model, SAFE_WEIGHTS_INDEX_NAME)):
|
||||
files = filter_duplicate_safetensors_files(
|
||||
@@ -337,8 +325,6 @@ class TensorEntry:
|
||||
|
||||
@classmethod
|
||||
def from_tensor(cls, tensor: torch.Tensor, kind: str) -> "TensorEntry":
|
||||
from torch.multiprocessing.reductions import reduce_tensor
|
||||
|
||||
tensor = tensor.detach()
|
||||
if tensor.is_cuda:
|
||||
_, ipc_args = reduce_tensor(tensor)
|
||||
@@ -349,8 +335,6 @@ class TensorEntry:
|
||||
if self.ipc_args is None:
|
||||
assert self.cpu_tensor is not None
|
||||
return self.cpu_tensor
|
||||
from torch.multiprocessing.reductions import rebuild_cuda_tensor
|
||||
|
||||
args = list(self.ipc_args)
|
||||
# Index 6 of the args from reduce_tensor is the device index. It must
|
||||
# be retargeted to the local index since the daemon and the engine may
|
||||
|
||||
Reference in New Issue
Block a user