[misc][plugin] add plugin system implementation (#7426)
This commit is contained in:
@@ -227,6 +227,9 @@ class LLMEngine:
|
||||
)
|
||||
# TODO(woosuk): Print more configs in debug mode.
|
||||
|
||||
from vllm.plugins import load_general_plugins
|
||||
load_general_plugins()
|
||||
|
||||
self.model_config = model_config
|
||||
self.cache_config = cache_config
|
||||
self.lora_config = lora_config
|
||||
|
||||
+9
-1
@@ -1,6 +1,6 @@
|
||||
import os
|
||||
import tempfile
|
||||
from typing import TYPE_CHECKING, Any, Callable, Dict, Optional
|
||||
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional
|
||||
|
||||
if TYPE_CHECKING:
|
||||
VLLM_HOST_IP: str = ""
|
||||
@@ -55,6 +55,7 @@ if TYPE_CHECKING:
|
||||
VERBOSE: bool = False
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: bool = False
|
||||
VLLM_TEST_FORCE_FP8_MARLIN: bool = False
|
||||
VLLM_PLUGINS: Optional[List[str]] = None
|
||||
|
||||
|
||||
def get_default_cache_root():
|
||||
@@ -362,6 +363,13 @@ environment_variables: Dict[str, Callable[[], Any]] = {
|
||||
lambda:
|
||||
(os.environ.get("VLLM_TEST_FORCE_FP8_MARLIN", "0").strip().lower() in
|
||||
("1", "true")),
|
||||
|
||||
# a list of plugin names to load, separated by commas.
|
||||
# if this is not set, it means all plugins will be loaded
|
||||
# if this is set to an empty string, no plugins will be loaded
|
||||
"VLLM_PLUGINS":
|
||||
lambda: None if "VLLM_PLUGINS" not in os.environ else os.environ[
|
||||
"VLLM_PLUGINS"].split(","),
|
||||
}
|
||||
|
||||
# end-env-vars-definition
|
||||
|
||||
@@ -166,7 +166,7 @@ class ModelRegistry:
|
||||
|
||||
@staticmethod
|
||||
def get_supported_archs() -> List[str]:
|
||||
return list(_MODELS.keys())
|
||||
return list(_MODELS.keys()) + list(_OOT_MODELS.keys())
|
||||
|
||||
@staticmethod
|
||||
def register_model(model_arch: str, model_cls: Type[nn.Module]):
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
import logging
|
||||
|
||||
import vllm.envs as envs
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def load_general_plugins():
|
||||
"""WARNING: plugins can be loaded for multiple times in different
|
||||
processes. They should be designed in a way that they can be loaded
|
||||
multiple times without causing issues.
|
||||
"""
|
||||
import sys
|
||||
if sys.version_info < (3, 10):
|
||||
from importlib_metadata import entry_points
|
||||
else:
|
||||
from importlib.metadata import entry_points
|
||||
|
||||
allowed_plugins = envs.VLLM_PLUGINS
|
||||
|
||||
discovered_plugins = entry_points(group='vllm.general_plugins')
|
||||
for plugin in discovered_plugins:
|
||||
logger.info("Found general plugin: %s", plugin.name)
|
||||
if allowed_plugins is None or plugin.name in allowed_plugins:
|
||||
try:
|
||||
func = plugin.load()
|
||||
func()
|
||||
logger.info("Loaded general plugin: %s", plugin.name)
|
||||
except Exception:
|
||||
logger.exception("Failed to load general plugin: %s",
|
||||
plugin.name)
|
||||
@@ -411,6 +411,9 @@ class WorkerWrapperBase:
|
||||
# see https://github.com/NVIDIA/nccl/issues/1234
|
||||
os.environ['NCCL_CUMEM_ENABLE'] = '0'
|
||||
|
||||
from vllm.plugins import load_general_plugins
|
||||
load_general_plugins()
|
||||
|
||||
if self.worker_class_fn:
|
||||
worker_class = self.worker_class_fn()
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user