[Frontend] Split OpenAIServingModels into OpenAIModelRegistry + OpenAIServingModels (#36536)
Signed-off-by: Sage Ahrac <sagiahrak@gmail.com>
This commit is contained in:
@@ -5,6 +5,7 @@ from asyncio import Lock
|
||||
from collections import defaultdict
|
||||
from http import HTTPStatus
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
@@ -27,6 +28,51 @@ from vllm.utils.counter import AtomicCounter
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class OpenAIModelRegistry:
|
||||
"""Read-only view of the loaded base models with no engine dependency.
|
||||
|
||||
Suitable for CPU-only / render-only contexts that have no engine client
|
||||
and no LoRA support.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_config: ModelConfig,
|
||||
base_model_paths: list[BaseModelPath],
|
||||
) -> None:
|
||||
self.model_config = model_config
|
||||
self.base_model_paths = base_model_paths
|
||||
|
||||
def is_base_model(self, model_name: str) -> bool:
|
||||
return any(model.name == model_name for model in self.base_model_paths)
|
||||
|
||||
async def check_model(self, model_name: str | None) -> ErrorResponse | None:
|
||||
"""Return an ErrorResponse if model_name is not served, else None."""
|
||||
if not model_name or self.is_base_model(model_name):
|
||||
return None
|
||||
return create_error_response(
|
||||
message=f"The model `{model_name}` does not exist.",
|
||||
err_type="NotFoundError",
|
||||
status_code=HTTPStatus.NOT_FOUND,
|
||||
param="model",
|
||||
)
|
||||
|
||||
async def show_available_models(self) -> ModelList:
|
||||
"""Show available models (base models only)."""
|
||||
max_model_len = self.model_config.max_model_len
|
||||
return ModelList(
|
||||
data=[
|
||||
ModelCard(
|
||||
id=base_model.name,
|
||||
max_model_len=max_model_len,
|
||||
root=base_model.model_path,
|
||||
permission=[ModelPermission()],
|
||||
)
|
||||
for base_model in self.base_model_paths
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
class OpenAIServingModels:
|
||||
"""Shared instance to hold data about the loaded base model(s) and adapters.
|
||||
|
||||
@@ -45,6 +91,11 @@ class OpenAIServingModels:
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
self.registry = OpenAIModelRegistry(
|
||||
model_config=engine_client.model_config,
|
||||
base_model_paths=base_model_paths,
|
||||
)
|
||||
|
||||
self.engine_client = engine_client
|
||||
self.base_model_paths = base_model_paths
|
||||
|
||||
@@ -79,34 +130,18 @@ class OpenAIServingModels:
|
||||
if isinstance(load_result, ErrorResponse):
|
||||
raise ValueError(load_result.error.message)
|
||||
|
||||
def is_base_model(self, model_name) -> bool:
|
||||
return any(model.name == model_name for model in self.base_model_paths)
|
||||
def is_base_model(self, model_name: str) -> bool:
|
||||
return self.registry.is_base_model(model_name)
|
||||
|
||||
def model_name(self, lora_request: LoRARequest | None = None) -> str:
|
||||
"""Returns the appropriate model name depending on the availability
|
||||
and support of the LoRA or base model.
|
||||
Parameters:
|
||||
- lora: LoRARequest that contain a base_model_name.
|
||||
Returns:
|
||||
- str: The name of the base model or the first available model path.
|
||||
"""
|
||||
if lora_request is not None:
|
||||
return lora_request.lora_name
|
||||
return self.base_model_paths[0].name
|
||||
|
||||
async def show_available_models(self) -> ModelList:
|
||||
"""Show available models. This includes the base model and all adapters."""
|
||||
max_model_len = self.model_config.max_model_len
|
||||
|
||||
model_cards = [
|
||||
ModelCard(
|
||||
id=base_model.name,
|
||||
max_model_len=max_model_len,
|
||||
root=base_model.model_path,
|
||||
permission=[ModelPermission()],
|
||||
)
|
||||
for base_model in self.base_model_paths
|
||||
]
|
||||
"""Show available models. This includes the base model and all
|
||||
adapters."""
|
||||
model_list = await self.registry.show_available_models()
|
||||
lora_cards = [
|
||||
ModelCard(
|
||||
id=lora.lora_name,
|
||||
@@ -118,8 +153,8 @@ class OpenAIServingModels:
|
||||
)
|
||||
for lora in self.lora_requests.values()
|
||||
]
|
||||
model_cards.extend(lora_cards)
|
||||
return ModelList(data=model_cards)
|
||||
model_list.data.extend(lora_cards)
|
||||
return model_list
|
||||
|
||||
async def load_lora_adapter(
|
||||
self, request: LoadLoRAAdapterRequest, base_model_name: str | None = None
|
||||
|
||||
Reference in New Issue
Block a user