[misc][plugin] add plugin system implementation (#7426)

This commit is contained in:
youkaichao
2024-08-13 16:24:17 -07:00
committed by GitHub
parent 373538f973
commit 16422ea76f
13 changed files with 162 additions and 102 deletions
+3
View File
@@ -227,6 +227,9 @@ class LLMEngine:
)
# TODO(woosuk): Print more configs in debug mode.
from vllm.plugins import load_general_plugins
load_general_plugins()
self.model_config = model_config
self.cache_config = cache_config
self.lora_config = lora_config
+9 -1
View File
@@ -1,6 +1,6 @@
import os
import tempfile
from typing import TYPE_CHECKING, Any, Callable, Dict, Optional
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional
if TYPE_CHECKING:
VLLM_HOST_IP: str = ""
@@ -55,6 +55,7 @@ if TYPE_CHECKING:
VERBOSE: bool = False
VLLM_ALLOW_LONG_MAX_MODEL_LEN: bool = False
VLLM_TEST_FORCE_FP8_MARLIN: bool = False
VLLM_PLUGINS: Optional[List[str]] = None
def get_default_cache_root():
@@ -362,6 +363,13 @@ environment_variables: Dict[str, Callable[[], Any]] = {
lambda:
(os.environ.get("VLLM_TEST_FORCE_FP8_MARLIN", "0").strip().lower() in
("1", "true")),
# a list of plugin names to load, separated by commas.
# if this is not set, it means all plugins will be loaded
# if this is set to an empty string, no plugins will be loaded
"VLLM_PLUGINS":
lambda: None if "VLLM_PLUGINS" not in os.environ else os.environ[
"VLLM_PLUGINS"].split(","),
}
# end-env-vars-definition
+1 -1
View File
@@ -166,7 +166,7 @@ class ModelRegistry:
@staticmethod
def get_supported_archs() -> List[str]:
return list(_MODELS.keys())
return list(_MODELS.keys()) + list(_OOT_MODELS.keys())
@staticmethod
def register_model(model_arch: str, model_cls: Type[nn.Module]):
+31
View File
@@ -0,0 +1,31 @@
import logging
import vllm.envs as envs
logger = logging.getLogger(__name__)
def load_general_plugins():
"""WARNING: plugins can be loaded for multiple times in different
processes. They should be designed in a way that they can be loaded
multiple times without causing issues.
"""
import sys
if sys.version_info < (3, 10):
from importlib_metadata import entry_points
else:
from importlib.metadata import entry_points
allowed_plugins = envs.VLLM_PLUGINS
discovered_plugins = entry_points(group='vllm.general_plugins')
for plugin in discovered_plugins:
logger.info("Found general plugin: %s", plugin.name)
if allowed_plugins is None or plugin.name in allowed_plugins:
try:
func = plugin.load()
func()
logger.info("Loaded general plugin: %s", plugin.name)
except Exception:
logger.exception("Failed to load general plugin: %s",
plugin.name)
+3
View File
@@ -411,6 +411,9 @@ class WorkerWrapperBase:
# see https://github.com/NVIDIA/nccl/issues/1234
os.environ['NCCL_CUMEM_ENABLE'] = '0'
from vllm.plugins import load_general_plugins
load_general_plugins()
if self.worker_class_fn:
worker_class = self.worker_class_fn()
else: