[platforms] enable platform plugins (#11602)
Signed-off-by: youkaichao <youkaichao@gmail.com>
This commit is contained in:
+1
-1
@@ -31,7 +31,6 @@ from vllm.inputs import (ExplicitEncoderDecoderPrompt, TextPrompt,
|
||||
to_enc_dec_tuple_list, zip_enc_dec_prompts)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sampling_params import BeamSearchParams
|
||||
from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, cuda_device_count_stateless,
|
||||
identity)
|
||||
@@ -242,6 +241,7 @@ _T = TypeVar("_T", nn.Module, torch.Tensor, BatchEncoding, BatchFeature, dict)
|
||||
class HfRunner:
|
||||
|
||||
def wrap_device(self, x: _T, device: Optional[str] = None) -> _T:
|
||||
from vllm.platforms import current_platform
|
||||
if x is None or isinstance(x, (bool, )):
|
||||
return x
|
||||
|
||||
|
||||
@@ -5,7 +5,10 @@ import torch
|
||||
|
||||
from tests.kernels.utils import override_backend_env_variable
|
||||
from vllm.attention.selector import which_attn_to_use
|
||||
from vllm.platforms import cpu, cuda, openvino, rocm
|
||||
from vllm.platforms.cpu import CpuPlatform
|
||||
from vllm.platforms.cuda import CudaPlatform
|
||||
from vllm.platforms.openvino import OpenVinoPlatform
|
||||
from vllm.platforms.rocm import RocmPlatform
|
||||
from vllm.utils import STR_FLASH_ATTN_VAL, STR_INVALID_VAL
|
||||
|
||||
|
||||
@@ -20,26 +23,23 @@ def test_env(name: str, device: str, monkeypatch):
|
||||
override_backend_env_variable(monkeypatch, name)
|
||||
|
||||
if device == "cpu":
|
||||
with patch("vllm.attention.selector.current_platform",
|
||||
cpu.CpuPlatform()):
|
||||
with patch("vllm.attention.selector.current_platform", CpuPlatform()):
|
||||
backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
|
||||
False)
|
||||
assert backend.name == "TORCH_SDPA"
|
||||
elif device == "hip":
|
||||
with patch("vllm.attention.selector.current_platform",
|
||||
rocm.RocmPlatform()):
|
||||
with patch("vllm.attention.selector.current_platform", RocmPlatform()):
|
||||
backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
|
||||
False)
|
||||
assert backend.name == "ROCM_FLASH"
|
||||
elif device == "openvino":
|
||||
with patch("vllm.attention.selector.current_platform",
|
||||
openvino.OpenVinoPlatform()):
|
||||
OpenVinoPlatform()):
|
||||
backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
|
||||
False)
|
||||
assert backend.name == "OPENVINO"
|
||||
else:
|
||||
with patch("vllm.attention.selector.current_platform",
|
||||
cuda.CudaPlatform()):
|
||||
with patch("vllm.attention.selector.current_platform", CudaPlatform()):
|
||||
backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
|
||||
False)
|
||||
assert backend.name == name
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
from setuptools import setup
|
||||
|
||||
setup(
|
||||
name='vllm_add_dummy_platform',
|
||||
version='0.1',
|
||||
packages=['vllm_add_dummy_platform'],
|
||||
entry_points={
|
||||
'vllm.platform_plugins': [
|
||||
"dummy_platform_plugin = vllm_add_dummy_platform:dummy_platform_plugin" # noqa
|
||||
]
|
||||
})
|
||||
@@ -0,0 +1,5 @@
|
||||
from typing import Optional
|
||||
|
||||
|
||||
def dummy_platform_plugin() -> Optional[str]:
|
||||
return "vllm_add_dummy_platform.dummy_platform.DummyPlatform"
|
||||
@@ -0,0 +1,5 @@
|
||||
from vllm.platforms.cuda import CudaPlatform
|
||||
|
||||
|
||||
class DummyPlatform(CudaPlatform):
|
||||
device_name = "DummyDevice"
|
||||
@@ -0,0 +1,16 @@
|
||||
def test_platform_plugins():
|
||||
# simulate workload by running an example
|
||||
import runpy
|
||||
current_file = __file__
|
||||
import os
|
||||
example_file = os.path.join(
|
||||
os.path.dirname(os.path.dirname(os.path.dirname(current_file))),
|
||||
"examples", "offline_inference.py")
|
||||
runpy.run_path(example_file)
|
||||
|
||||
# check if the plugin is loaded correctly
|
||||
from vllm.platforms import _init_trace, current_platform
|
||||
assert current_platform.device_name == "DummyDevice", (
|
||||
f"Expected DummyDevice, got {current_platform.device_name}, "
|
||||
"possibly because current_platform is imported before the plugin"
|
||||
f" is loaded. The first import:\n{_init_trace}")
|
||||
Reference in New Issue
Block a user