diff --git a/tests/model_executor/model_loader/test_weight_cache.py b/tests/model_executor/model_loader/test_weight_cache.py index 6084d5e7ed..c1b2e252eb 100644 --- a/tests/model_executor/model_loader/test_weight_cache.py +++ b/tests/model_executor/model_loader/test_weight_cache.py @@ -14,7 +14,10 @@ import tempfile import threading import time +import pytest + from vllm import SamplingParams +from vllm.platforms import current_platform MODEL = "Qwen/Qwen3.5-0.8B" PROMPTS = [ @@ -121,6 +124,8 @@ def test_ipc_cache_cold_start_and_warm_restart(vllm_runner): warm runs disable the disk fallback, so they only pass if the weights really came from the daemon. """ + if not current_platform.is_cuda_alike(): + pytest.skip("Weight cache IPC sharing requires CUDA or ROCm") llm_kwargs = dict( gpu_memory_utilization=0.3, enforce_eager=True, diff --git a/vllm/model_executor/model_loader/weight_cache/__init__.py b/vllm/model_executor/model_loader/weight_cache/__init__.py index 022770f893..0cea5cf72a 100644 --- a/vllm/model_executor/model_loader/weight_cache/__init__.py +++ b/vllm/model_executor/model_loader/weight_cache/__init__.py @@ -5,9 +5,11 @@ from vllm.model_executor.model_loader.weight_cache.ipc_loader import IpcModelLoa from vllm.model_executor.model_loader.weight_cache.protocol import ( CacheConfigMismatchError, TensorEntry, + UnsupportedPlatformForIPCError, UnsupportedQuantForIPCError, WeightCacheKey, WeightCacheUnavailableError, + check_ipc_platform_support, check_ipc_quant_support, is_ipc_quant_supported, ) @@ -16,9 +18,11 @@ __all__ = [ "CacheConfigMismatchError", "IpcModelLoader", "TensorEntry", + "UnsupportedPlatformForIPCError", "UnsupportedQuantForIPCError", "WeightCacheKey", "WeightCacheUnavailableError", + "check_ipc_platform_support", "check_ipc_quant_support", "is_ipc_quant_supported", ] diff --git a/vllm/model_executor/model_loader/weight_cache/daemon.py b/vllm/model_executor/model_loader/weight_cache/daemon.py index 7bb10b936e..941f63de62 100644 --- a/vllm/model_executor/model_loader/weight_cache/daemon.py +++ b/vllm/model_executor/model_loader/weight_cache/daemon.py @@ -43,6 +43,7 @@ from vllm.model_executor.model_loader.weight_cache.protocol import ( TensorEntry, WeightCacheKey, WeightCacheUnavailableError, + check_ipc_platform_support, check_ipc_quant_support, ensure_private_socket_dir, get_physical_device_id, @@ -205,12 +206,14 @@ class WeightCacheDaemon: self._handle_connection(conn) except (ConnectionError, EOFError): logger.warning("Client disconnected mid-request") - except Exception: - # A single malformed or malicious request must not take - # down the daemon for every other engine on this GPU. + except Exception as e: + # Report the error back instead of just closing the + # socket, but don't let it take the daemon down. logger.exception( "Error handling weight cache client; continuing" ) + with contextlib.suppress(OSError): + send_msg(conn, {"status": "error", "message": str(e)}) finally: server.close() if os.path.exists(socket_path): @@ -344,6 +347,7 @@ def main() -> None: ) # Checked before loading anything: an unsupported quantization method would # otherwise only surface in the engine, after a full load. + check_ipc_platform_support(where="daemon") check_ipc_quant_support(vllm_config.model_config, where="daemon") parallel_config = vllm_config.parallel_config _reject_unsupported_parallelism(parallel_config) diff --git a/vllm/model_executor/model_loader/weight_cache/ipc_loader.py b/vllm/model_executor/model_loader/weight_cache/ipc_loader.py index 93c5bfc016..2132a55fbe 100644 --- a/vllm/model_executor/model_loader/weight_cache/ipc_loader.py +++ b/vllm/model_executor/model_loader/weight_cache/ipc_loader.py @@ -25,6 +25,7 @@ from vllm.model_executor.model_loader.weight_cache.protocol import ( UnsupportedQuantForIPCError, WeightCacheKey, WeightCacheUnavailableError, + check_ipc_platform_support, check_ipc_quant_support, get_physical_device_id, get_socket_path, @@ -198,6 +199,7 @@ class IpcModelLoader(BaseModelLoader): @staticmethod def _check_supported(vllm_config: VllmConfig, model_config: ModelConfig) -> None: + check_ipc_platform_support(where="engine") check_ipc_quant_support(model_config, where="engine") cache_dtype = vllm_config.cache_config.cache_dtype if cache_dtype != "auto" and not str(cache_dtype).startswith("fp8"): diff --git a/vllm/model_executor/model_loader/weight_cache/protocol.py b/vllm/model_executor/model_loader/weight_cache/protocol.py index 3866cc773b..ae5bb3b3b3 100644 --- a/vllm/model_executor/model_loader/weight_cache/protocol.py +++ b/vllm/model_executor/model_loader/weight_cache/protocol.py @@ -26,6 +26,7 @@ import torch import vllm.version from vllm.config import ModelConfig +from vllm.platforms import current_platform from vllm.utils.hashing import safe_hash SOCKET_NAME_TEMPLATE = "vllm_weight_cache_gpu{gpu_id}.sock" @@ -54,6 +55,32 @@ class UnsupportedQuantForIPCError(Exception): """Raised when a quantization method is not verified for IPC weight sharing.""" +class UnsupportedPlatformForIPCError(Exception): + """Raised when the current platform cannot share CUDA IPC handles.""" + + +def check_ipc_platform_support(*, where: str) -> None: + """Hard-error unless the current platform can share CUDA IPC handles. + + Only CUDA/ROCm tensors get a real IPC handle from ``TensorEntry``; other + platforms (e.g. XPU) would silently ship every tensor by value instead. + + Args: + where: Short tag ("daemon"/"engine") used in the error message. + + Raises: + UnsupportedPlatformForIPCError: If the current platform is not + CUDA/ROCm. + """ + if current_platform.is_cuda_alike(): + return + raise UnsupportedPlatformForIPCError( + f"[weight_cache:{where}] platform {current_platform.device_name!r} " + "does not support CUDA IPC weight sharing; only CUDA and ROCm are " + "supported. Use the default --load-format for this platform." + ) + + # The daemon exports tensor data only, so sharing is correct just for methods # whose post-load effect is either fully captured by that data or rebuilt when # process_weights_after_loading runs under weights_already_processed. Anything