[CI][XPU] Reject CUDA-IPC weight cache on non-CUDA/ROCm platforms (#56010)
Signed-off-by: mayuyuace <qiming1.zhang@intel.com>
This commit is contained in:
@@ -14,7 +14,10 @@ import tempfile
|
|||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
from vllm import SamplingParams
|
from vllm import SamplingParams
|
||||||
|
from vllm.platforms import current_platform
|
||||||
|
|
||||||
MODEL = "Qwen/Qwen3.5-0.8B"
|
MODEL = "Qwen/Qwen3.5-0.8B"
|
||||||
PROMPTS = [
|
PROMPTS = [
|
||||||
@@ -121,6 +124,8 @@ def test_ipc_cache_cold_start_and_warm_restart(vllm_runner):
|
|||||||
warm runs disable the disk fallback, so they only pass if the weights
|
warm runs disable the disk fallback, so they only pass if the weights
|
||||||
really came from the daemon.
|
really came from the daemon.
|
||||||
"""
|
"""
|
||||||
|
if not current_platform.is_cuda_alike():
|
||||||
|
pytest.skip("Weight cache IPC sharing requires CUDA or ROCm")
|
||||||
llm_kwargs = dict(
|
llm_kwargs = dict(
|
||||||
gpu_memory_utilization=0.3,
|
gpu_memory_utilization=0.3,
|
||||||
enforce_eager=True,
|
enforce_eager=True,
|
||||||
|
|||||||
@@ -5,9 +5,11 @@ from vllm.model_executor.model_loader.weight_cache.ipc_loader import IpcModelLoa
|
|||||||
from vllm.model_executor.model_loader.weight_cache.protocol import (
|
from vllm.model_executor.model_loader.weight_cache.protocol import (
|
||||||
CacheConfigMismatchError,
|
CacheConfigMismatchError,
|
||||||
TensorEntry,
|
TensorEntry,
|
||||||
|
UnsupportedPlatformForIPCError,
|
||||||
UnsupportedQuantForIPCError,
|
UnsupportedQuantForIPCError,
|
||||||
WeightCacheKey,
|
WeightCacheKey,
|
||||||
WeightCacheUnavailableError,
|
WeightCacheUnavailableError,
|
||||||
|
check_ipc_platform_support,
|
||||||
check_ipc_quant_support,
|
check_ipc_quant_support,
|
||||||
is_ipc_quant_supported,
|
is_ipc_quant_supported,
|
||||||
)
|
)
|
||||||
@@ -16,9 +18,11 @@ __all__ = [
|
|||||||
"CacheConfigMismatchError",
|
"CacheConfigMismatchError",
|
||||||
"IpcModelLoader",
|
"IpcModelLoader",
|
||||||
"TensorEntry",
|
"TensorEntry",
|
||||||
|
"UnsupportedPlatformForIPCError",
|
||||||
"UnsupportedQuantForIPCError",
|
"UnsupportedQuantForIPCError",
|
||||||
"WeightCacheKey",
|
"WeightCacheKey",
|
||||||
"WeightCacheUnavailableError",
|
"WeightCacheUnavailableError",
|
||||||
|
"check_ipc_platform_support",
|
||||||
"check_ipc_quant_support",
|
"check_ipc_quant_support",
|
||||||
"is_ipc_quant_supported",
|
"is_ipc_quant_supported",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -43,6 +43,7 @@ from vllm.model_executor.model_loader.weight_cache.protocol import (
|
|||||||
TensorEntry,
|
TensorEntry,
|
||||||
WeightCacheKey,
|
WeightCacheKey,
|
||||||
WeightCacheUnavailableError,
|
WeightCacheUnavailableError,
|
||||||
|
check_ipc_platform_support,
|
||||||
check_ipc_quant_support,
|
check_ipc_quant_support,
|
||||||
ensure_private_socket_dir,
|
ensure_private_socket_dir,
|
||||||
get_physical_device_id,
|
get_physical_device_id,
|
||||||
@@ -205,12 +206,14 @@ class WeightCacheDaemon:
|
|||||||
self._handle_connection(conn)
|
self._handle_connection(conn)
|
||||||
except (ConnectionError, EOFError):
|
except (ConnectionError, EOFError):
|
||||||
logger.warning("Client disconnected mid-request")
|
logger.warning("Client disconnected mid-request")
|
||||||
except Exception:
|
except Exception as e:
|
||||||
# A single malformed or malicious request must not take
|
# Report the error back instead of just closing the
|
||||||
# down the daemon for every other engine on this GPU.
|
# socket, but don't let it take the daemon down.
|
||||||
logger.exception(
|
logger.exception(
|
||||||
"Error handling weight cache client; continuing"
|
"Error handling weight cache client; continuing"
|
||||||
)
|
)
|
||||||
|
with contextlib.suppress(OSError):
|
||||||
|
send_msg(conn, {"status": "error", "message": str(e)})
|
||||||
finally:
|
finally:
|
||||||
server.close()
|
server.close()
|
||||||
if os.path.exists(socket_path):
|
if os.path.exists(socket_path):
|
||||||
@@ -344,6 +347,7 @@ def main() -> None:
|
|||||||
)
|
)
|
||||||
# Checked before loading anything: an unsupported quantization method would
|
# Checked before loading anything: an unsupported quantization method would
|
||||||
# otherwise only surface in the engine, after a full load.
|
# otherwise only surface in the engine, after a full load.
|
||||||
|
check_ipc_platform_support(where="daemon")
|
||||||
check_ipc_quant_support(vllm_config.model_config, where="daemon")
|
check_ipc_quant_support(vllm_config.model_config, where="daemon")
|
||||||
parallel_config = vllm_config.parallel_config
|
parallel_config = vllm_config.parallel_config
|
||||||
_reject_unsupported_parallelism(parallel_config)
|
_reject_unsupported_parallelism(parallel_config)
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ from vllm.model_executor.model_loader.weight_cache.protocol import (
|
|||||||
UnsupportedQuantForIPCError,
|
UnsupportedQuantForIPCError,
|
||||||
WeightCacheKey,
|
WeightCacheKey,
|
||||||
WeightCacheUnavailableError,
|
WeightCacheUnavailableError,
|
||||||
|
check_ipc_platform_support,
|
||||||
check_ipc_quant_support,
|
check_ipc_quant_support,
|
||||||
get_physical_device_id,
|
get_physical_device_id,
|
||||||
get_socket_path,
|
get_socket_path,
|
||||||
@@ -198,6 +199,7 @@ class IpcModelLoader(BaseModelLoader):
|
|||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _check_supported(vllm_config: VllmConfig, model_config: ModelConfig) -> None:
|
def _check_supported(vllm_config: VllmConfig, model_config: ModelConfig) -> None:
|
||||||
|
check_ipc_platform_support(where="engine")
|
||||||
check_ipc_quant_support(model_config, where="engine")
|
check_ipc_quant_support(model_config, where="engine")
|
||||||
cache_dtype = vllm_config.cache_config.cache_dtype
|
cache_dtype = vllm_config.cache_config.cache_dtype
|
||||||
if cache_dtype != "auto" and not str(cache_dtype).startswith("fp8"):
|
if cache_dtype != "auto" and not str(cache_dtype).startswith("fp8"):
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ import torch
|
|||||||
|
|
||||||
import vllm.version
|
import vllm.version
|
||||||
from vllm.config import ModelConfig
|
from vllm.config import ModelConfig
|
||||||
|
from vllm.platforms import current_platform
|
||||||
from vllm.utils.hashing import safe_hash
|
from vllm.utils.hashing import safe_hash
|
||||||
|
|
||||||
SOCKET_NAME_TEMPLATE = "vllm_weight_cache_gpu{gpu_id}.sock"
|
SOCKET_NAME_TEMPLATE = "vllm_weight_cache_gpu{gpu_id}.sock"
|
||||||
@@ -54,6 +55,32 @@ class UnsupportedQuantForIPCError(Exception):
|
|||||||
"""Raised when a quantization method is not verified for IPC weight sharing."""
|
"""Raised when a quantization method is not verified for IPC weight sharing."""
|
||||||
|
|
||||||
|
|
||||||
|
class UnsupportedPlatformForIPCError(Exception):
|
||||||
|
"""Raised when the current platform cannot share CUDA IPC handles."""
|
||||||
|
|
||||||
|
|
||||||
|
def check_ipc_platform_support(*, where: str) -> None:
|
||||||
|
"""Hard-error unless the current platform can share CUDA IPC handles.
|
||||||
|
|
||||||
|
Only CUDA/ROCm tensors get a real IPC handle from ``TensorEntry``; other
|
||||||
|
platforms (e.g. XPU) would silently ship every tensor by value instead.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
where: Short tag ("daemon"/"engine") used in the error message.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
UnsupportedPlatformForIPCError: If the current platform is not
|
||||||
|
CUDA/ROCm.
|
||||||
|
"""
|
||||||
|
if current_platform.is_cuda_alike():
|
||||||
|
return
|
||||||
|
raise UnsupportedPlatformForIPCError(
|
||||||
|
f"[weight_cache:{where}] platform {current_platform.device_name!r} "
|
||||||
|
"does not support CUDA IPC weight sharing; only CUDA and ROCm are "
|
||||||
|
"supported. Use the default --load-format for this platform."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# The daemon exports tensor data only, so sharing is correct just for methods
|
# The daemon exports tensor data only, so sharing is correct just for methods
|
||||||
# whose post-load effect is either fully captured by that data or rebuilt when
|
# whose post-load effect is either fully captured by that data or rebuilt when
|
||||||
# process_weights_after_loading runs under weights_already_processed. Anything
|
# process_weights_after_loading runs under weights_already_processed. Anything
|
||||||
|
|||||||
Reference in New Issue
Block a user