[Bugfix][Platform] Don't resolve current_platform during import vllm (#60766)
Signed-off-by: Haotian An <haotiaa@amazon.com>
This commit is contained in:
@@ -3,6 +3,9 @@
|
||||
|
||||
|
||||
def dummy_platform_plugin() -> str | None:
|
||||
# Real platform plugins often import vLLM modules while registering.
|
||||
from vllm.utils.torch_utils import direct_register_custom_op # noqa: F401
|
||||
|
||||
return "vllm_add_dummy_platform.dummy_platform.DummyPlatform"
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -31,6 +34,20 @@ def test_platform_plugins():
|
||||
)
|
||||
|
||||
|
||||
def test_import_vllm_does_not_resolve_platform():
|
||||
# Platform plugins must be loaded after `import vllm` completes; resolving
|
||||
# current_platform during it runs plugins against a half-imported vLLM.
|
||||
# Needs a fresh interpreter, since vllm is already imported here.
|
||||
code = (
|
||||
"import sys, vllm\n"
|
||||
"p = sys.modules.get('vllm.platforms')\n"
|
||||
"assert p is None or p._current_platform is None, p._init_trace\n"
|
||||
"from vllm.platforms import current_platform\n"
|
||||
"assert current_platform.device_name == 'DummyDevice'\n"
|
||||
)
|
||||
subprocess.run([sys.executable, "-c", code], check=True)
|
||||
|
||||
|
||||
def test_oot_custom_op(default_vllm_config, monkeypatch: pytest.MonkeyPatch):
|
||||
# simulate workload by running an example
|
||||
load_general_plugins()
|
||||
|
||||
@@ -73,7 +73,15 @@ MODELOPT_TO_VLLM_KV_CACHE_DTYPE_MAP = {
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
PIN_MEMORY = is_pin_memory_available()
|
||||
if TYPE_CHECKING:
|
||||
PIN_MEMORY: bool
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any:
|
||||
# Lazy so that importing this module does not resolve current_platform.
|
||||
if name == "PIN_MEMORY":
|
||||
return is_pin_memory_available()
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
|
||||
def is_quantized_kv_cache(kv_cache_dtype: str) -> bool:
|
||||
@@ -718,13 +726,15 @@ def async_tensor_h2d(
|
||||
data = torch.from_numpy(data)
|
||||
if isinstance(data, torch.Tensor):
|
||||
t = data
|
||||
if PIN_MEMORY and not t.is_pinned():
|
||||
if is_pin_memory_available() and not t.is_pinned():
|
||||
# Stage in pinned, contiguous buffer to ensure fully async copy.
|
||||
t = torch.empty(
|
||||
t.shape, dtype=dtype or t.dtype, device="cpu", pin_memory=True
|
||||
).copy_(t)
|
||||
else:
|
||||
t = torch.tensor(data, dtype=dtype, pin_memory=PIN_MEMORY, device="cpu")
|
||||
t = torch.tensor(
|
||||
data, dtype=dtype, pin_memory=is_pin_memory_available(), device="cpu"
|
||||
)
|
||||
assert t.is_cpu
|
||||
|
||||
if out is not None:
|
||||
@@ -736,7 +746,7 @@ def async_tensor_h2d(
|
||||
|
||||
def np_to_pinned_tensor(array: np.ndarray) -> torch.Tensor:
|
||||
t = torch.from_numpy(array)
|
||||
return t.pin_memory() if PIN_MEMORY else t
|
||||
return t.pin_memory() if is_pin_memory_available() else t
|
||||
|
||||
|
||||
def make_ndarray_with_pad(
|
||||
|
||||
Reference in New Issue
Block a user