MOVZXandHinata
5aa2ac64f8
Production runs one static model behind the OpenAI-compatible API. Every adapter path and non-OpenAI HTTP surface is unreachable there. - Delete the LoRA stack: vllm/lora, LoRARequest on the engine wire, the scheduler, worker, cudagraph and MoE plumbing, LoRA metrics, the lora CLI group, and the LoRA tests. - Drop the plugin loader in vllm/plugins. Platform resolution is now CUDA-only and the general/stat-logger plugin hooks are gone. - Delete the speech-to-text and realtime entrypoints, the scale-out render routes, and vllm/config/speech_to_text.py. GenerationTask is generate-only. - Cut DecodeBenchConnector and the disagg-prefill doc and image. The Multi, Offloading and Example connectors stay; production uses the CPU KV offload tier. - Delete the ROCm-only fused KDA chunk kernel (2346 lines) with its CMake arch gate, declarations and op registrations. The CUDA build never defined the guard and no Python caller existed. - Delete the orphan Mistral and Pixtral chat fixtures and examples/. - Fix _check_model: removing the LoRA early return made every request fail with "The model ... does not exist. Valid aliases: ...". Verified on an RTX 3090: import sweep reports 1242 modules with 7 known optional-dependency failures; a live server answers chat, completions, responses, tokenize and detokenize with zero log errors and 404 for the removed surfaces; collection yields 8558 tests with the 3 pre-existing errors; ruff adds no new finding; the incremental _C build succeeds. Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
100 lines
3.1 KiB
Python
100 lines
3.1 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
from http import HTTPStatus
|
|
|
|
from vllm.config import ModelConfig
|
|
from vllm.engine.protocol import EngineClient
|
|
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
|
from vllm.entrypoints.serve import create_error_response
|
|
from vllm.entrypoints.serve.engine.protocol import (
|
|
ErrorResponse,
|
|
ModelCard,
|
|
ModelList,
|
|
ModelPermission,
|
|
)
|
|
|
|
|
|
class OpenAIModelRegistry:
|
|
"""Read-only view of the loaded base models with no engine dependency.
|
|
|
|
Suitable for CPU-only / render-only contexts that have no engine client.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
model_config: ModelConfig,
|
|
base_model_paths: list[BaseModelPath],
|
|
) -> None:
|
|
self.model_config = model_config
|
|
self.base_model_paths = base_model_paths
|
|
|
|
def model_name(self) -> str:
|
|
return self.base_model_paths[0].name
|
|
|
|
def is_base_model(self, model_name: str) -> bool:
|
|
return any(model.name == model_name for model in self.base_model_paths)
|
|
|
|
async def check_model(self, model_name: str | None) -> ErrorResponse | None:
|
|
"""Return an ErrorResponse if model_name is not served, else None."""
|
|
if not model_name or self.is_base_model(model_name):
|
|
return None
|
|
return create_error_response(
|
|
message=f"The model `{model_name}` does not exist.",
|
|
err_type="NotFoundError",
|
|
status_code=HTTPStatus.NOT_FOUND,
|
|
param="model",
|
|
)
|
|
|
|
async def show_available_models(self) -> ModelList:
|
|
"""Show available models (base models only)."""
|
|
max_model_len = self.model_config.max_model_len
|
|
return ModelList(
|
|
data=[
|
|
ModelCard(
|
|
id=base_model.name,
|
|
max_model_len=max_model_len,
|
|
root=base_model.model_path,
|
|
permission=[ModelPermission()],
|
|
)
|
|
for base_model in self.base_model_paths
|
|
]
|
|
)
|
|
|
|
|
|
class OpenAIServingModels:
|
|
"""Shared instance to hold data about the loaded base model(s).
|
|
|
|
Handles the routes:
|
|
- /v1/models
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
engine_client: EngineClient,
|
|
base_model_paths: list[BaseModelPath],
|
|
):
|
|
super().__init__()
|
|
|
|
self.registry = OpenAIModelRegistry(
|
|
model_config=engine_client.model_config,
|
|
base_model_paths=base_model_paths,
|
|
)
|
|
|
|
self.engine_client = engine_client
|
|
self.base_model_paths = base_model_paths
|
|
|
|
self.model_config = self.engine_client.model_config
|
|
self.renderer = self.engine_client.renderer
|
|
self.input_processor = self.engine_client.input_processor
|
|
|
|
def is_base_model(self, model_name: str) -> bool:
|
|
return self.registry.is_base_model(model_name)
|
|
|
|
def model_name(self) -> str:
|
|
return self.base_model_paths[0].name
|
|
|
|
async def show_available_models(self) -> ModelList:
|
|
"""Show available models."""
|
|
return await self.registry.show_available_models()
|