[Mini-VLLM] Remove LoRA, plugins, speech-to-text and scale-out surfaces
Production runs one static model behind the OpenAI-compatible API. Every adapter path and non-OpenAI HTTP surface is unreachable there. - Delete the LoRA stack: vllm/lora, LoRARequest on the engine wire, the scheduler, worker, cudagraph and MoE plumbing, LoRA metrics, the lora CLI group, and the LoRA tests. - Drop the plugin loader in vllm/plugins. Platform resolution is now CUDA-only and the general/stat-logger plugin hooks are gone. - Delete the speech-to-text and realtime entrypoints, the scale-out render routes, and vllm/config/speech_to_text.py. GenerationTask is generate-only. - Cut DecodeBenchConnector and the disagg-prefill doc and image. The Multi, Offloading and Example connectors stay; production uses the CPU KV offload tier. - Delete the ROCm-only fused KDA chunk kernel (2346 lines) with its CMake arch gate, declarations and op registrations. The CUDA build never defined the guard and no Python caller existed. - Delete the orphan Mistral and Pixtral chat fixtures and examples/. - Fix _check_model: removing the LoRA early return made every request fail with "The model ... does not exist. Valid aliases: ...". Verified on an RTX 3090: import sweep reports 1242 modules with 7 known optional-dependency failures; a live server answers chat, completions, responses, tokenize and detokenize with zero log errors and 404 for the removed surfaces; collection yields 8558 tests with the 3 pre-existing errors; ruff adds no new finding; the incremental _C build succeeds. Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
+1
-119
@@ -28,7 +28,6 @@ from typing import (
|
||||
|
||||
import huggingface_hub
|
||||
import regex as re
|
||||
import torch
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
from pydantic.fields import FieldInfo
|
||||
from typing_extensions import TypeIs
|
||||
@@ -52,7 +51,6 @@ from vllm.config import (
|
||||
KVTransferConfig,
|
||||
LoadConfig,
|
||||
LoggingConfig,
|
||||
LoRAConfig,
|
||||
MambaConfig,
|
||||
ModelConfig,
|
||||
MultiModalConfig,
|
||||
@@ -88,7 +86,6 @@ from vllm.config.kernel import (
|
||||
)
|
||||
from vllm.config.load import SafetensorsLoadStrategy
|
||||
from vllm.config.logging import LogLevel
|
||||
from vllm.config.lora import MaxLoRARanks
|
||||
from vllm.config.mamba import MambaBackendEnum, MambaSSUAlgorithm
|
||||
from vllm.config.model import (
|
||||
ConvertOption,
|
||||
@@ -119,7 +116,6 @@ from vllm.config.vllm import OptimizationLevel, PerformanceMode
|
||||
from vllm.config.watermarking import WatermarkConfig
|
||||
from vllm.logger import configure_logging_if_needed, init_logger, suppress_logging
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
from vllm.plugins import load_general_plugins
|
||||
from vllm.ray.lazy_utils import is_in_ray_actor, is_ray_initialized
|
||||
from vllm.transformers_utils.config import maybe_override_with_speculators
|
||||
from vllm.transformers_utils.repo_utils import get_model_path
|
||||
@@ -682,20 +678,6 @@ class EngineArgs:
|
||||
mm_processor_device: MMProcessorDevice = "auto"
|
||||
mm_ipc_gpu_memory_gb: float = MultiModalConfig.mm_ipc_gpu_memory_gb
|
||||
mm_device_do_normalize: bool | None = MultiModalConfig.mm_device_do_normalize
|
||||
# LoRA fields
|
||||
enable_lora: bool = False
|
||||
max_loras: int = LoRAConfig.max_loras
|
||||
max_lora_rank: MaxLoRARanks = LoRAConfig.max_lora_rank
|
||||
default_mm_loras: dict[str, str] | None = LoRAConfig.default_mm_loras
|
||||
fully_sharded_loras: bool = LoRAConfig.fully_sharded_loras
|
||||
max_cpu_loras: int | None = LoRAConfig.max_cpu_loras
|
||||
lora_dtype: str | torch.dtype | None = LoRAConfig.lora_dtype
|
||||
lora_target_modules: list[str] | None = LoRAConfig.target_modules
|
||||
enable_tower_connector_lora: bool = LoRAConfig.enable_tower_connector_lora
|
||||
specialize_active_lora: bool = LoRAConfig.specialize_active_lora
|
||||
enable_mixed_moe_lora_format: bool = LoRAConfig.enable_mixed_moe_lora_format
|
||||
enable_moe_shared_loras: bool = LoRAConfig.enable_moe_shared_loras
|
||||
max_lora_cls_labels: int | None = LoRAConfig.max_lora_cls_labels
|
||||
|
||||
ray_workers_use_nsight: bool = ParallelConfig.ray_workers_use_nsight
|
||||
num_gpu_blocks_override: int | None = CacheConfig.num_gpu_blocks_override
|
||||
@@ -848,7 +830,6 @@ class EngineArgs:
|
||||
|
||||
kv_offloading_size: float | None = CacheConfig.kv_offloading_size
|
||||
kv_offloading_backend: KVOffloadingBackend = CacheConfig.kv_offloading_backend
|
||||
tokens_only: bool = False
|
||||
|
||||
shutdown_timeout: int = 0
|
||||
|
||||
@@ -911,10 +892,6 @@ class EngineArgs:
|
||||
self.quantization, self.quantization_config
|
||||
)
|
||||
|
||||
# Setup plugins
|
||||
from vllm.plugins import load_general_plugins
|
||||
|
||||
load_general_plugins()
|
||||
# when use hf offline,replace model and tokenizer id to local model path
|
||||
if huggingface_hub.constants.HF_HUB_OFFLINE:
|
||||
# Skip cloud storage URIs (s3://, gs://, az://) — they are not
|
||||
@@ -1579,51 +1556,6 @@ class EngineArgs:
|
||||
},
|
||||
)
|
||||
|
||||
# LoRA related configs
|
||||
lora_kwargs = get_kwargs(LoRAConfig)
|
||||
lora_group = parser.add_argument_group(
|
||||
title="LoRAConfig",
|
||||
description=LoRAConfig.__doc__,
|
||||
)
|
||||
lora_group.add_argument(
|
||||
"--enable-lora",
|
||||
action=argparse.BooleanOptionalAction,
|
||||
help="If True, enable handling of LoRA adapters.",
|
||||
)
|
||||
lora_group.add_argument("--max-loras", **lora_kwargs["max_loras"])
|
||||
lora_group.add_argument("--max-lora-rank", **lora_kwargs["max_lora_rank"])
|
||||
lora_group.add_argument(
|
||||
"--lora-dtype",
|
||||
**lora_kwargs["lora_dtype"],
|
||||
)
|
||||
lora_group.add_argument(
|
||||
"--enable-tower-connector-lora",
|
||||
**lora_kwargs["enable_tower_connector_lora"],
|
||||
)
|
||||
lora_group.add_argument("--max-cpu-loras", **lora_kwargs["max_cpu_loras"])
|
||||
lora_group.add_argument(
|
||||
"--fully-sharded-loras", **lora_kwargs["fully_sharded_loras"]
|
||||
)
|
||||
lora_group.add_argument(
|
||||
"--lora-target-modules", **lora_kwargs["target_modules"]
|
||||
)
|
||||
lora_group.add_argument("--default-mm-loras", **lora_kwargs["default_mm_loras"])
|
||||
lora_group.add_argument(
|
||||
"--specialize-active-lora", **lora_kwargs["specialize_active_lora"]
|
||||
)
|
||||
lora_group.add_argument(
|
||||
"--enable-mixed-moe-lora-format",
|
||||
**lora_kwargs["enable_mixed_moe_lora_format"],
|
||||
)
|
||||
lora_group.add_argument(
|
||||
"--enable-moe-shared-loras",
|
||||
**lora_kwargs["enable_moe_shared_loras"],
|
||||
)
|
||||
lora_group.add_argument(
|
||||
"--max-lora-cls-labels",
|
||||
**lora_kwargs["max_lora_cls_labels"],
|
||||
)
|
||||
|
||||
# Logging arguments
|
||||
logging_group = parser.add_argument_group(
|
||||
title="LoggingConfig",
|
||||
@@ -2537,10 +2469,6 @@ class EngineArgs:
|
||||
else ParallelConfig.data_parallel_rpc_port
|
||||
)
|
||||
|
||||
if self.tokens_only and not model_config.skip_tokenizer_init:
|
||||
model_config.skip_tokenizer_init = True
|
||||
logger.info("Skipping tokenizer initialization for tokens-only mode.")
|
||||
|
||||
parallel_config = ParallelConfig(
|
||||
pipeline_parallel_size=self.pipeline_parallel_size,
|
||||
tensor_parallel_size=self.tensor_parallel_size,
|
||||
@@ -2657,47 +2585,6 @@ class EngineArgs:
|
||||
stream_interval=self.stream_interval,
|
||||
)
|
||||
|
||||
if not model_config.is_multimodal_model and self.default_mm_loras:
|
||||
raise ValueError(
|
||||
"Default modality-specific LoRA(s) were provided for a "
|
||||
"non multimodal model"
|
||||
)
|
||||
|
||||
lora_config = (
|
||||
LoRAConfig(
|
||||
max_lora_rank=self.max_lora_rank,
|
||||
max_loras=self.max_loras,
|
||||
default_mm_loras=self.default_mm_loras,
|
||||
fully_sharded_loras=self.fully_sharded_loras,
|
||||
lora_dtype=self.lora_dtype,
|
||||
target_modules=self.lora_target_modules,
|
||||
enable_tower_connector_lora=self.enable_tower_connector_lora,
|
||||
specialize_active_lora=self.specialize_active_lora,
|
||||
enable_mixed_moe_lora_format=self.enable_mixed_moe_lora_format,
|
||||
enable_moe_shared_loras=self.enable_moe_shared_loras,
|
||||
max_lora_cls_labels=self.max_lora_cls_labels,
|
||||
max_cpu_loras=self.max_cpu_loras
|
||||
if self.max_cpu_loras and self.max_cpu_loras > 0
|
||||
else None,
|
||||
)
|
||||
if self.enable_lora
|
||||
else None
|
||||
)
|
||||
|
||||
if (
|
||||
lora_config is not None
|
||||
and speculative_config is not None
|
||||
and scheduler_config.max_num_batched_tokens
|
||||
< (
|
||||
scheduler_config.max_num_seqs
|
||||
* (speculative_config.num_speculative_tokens + 1)
|
||||
)
|
||||
):
|
||||
raise ValueError(
|
||||
"Consider increasing max_num_batched_tokens or "
|
||||
"decreasing num_speculative_tokens"
|
||||
)
|
||||
|
||||
# Attention config overrides
|
||||
attention_config = copy.deepcopy(self.attention_config)
|
||||
if self.attention_backend is not None:
|
||||
@@ -2867,7 +2754,6 @@ class EngineArgs:
|
||||
engram_config=self.engram_config,
|
||||
mamba_config=mamba_config,
|
||||
kernel_config=kernel_config,
|
||||
lora_config=lora_config,
|
||||
speculative_config=speculative_config,
|
||||
watermark_config=watermark_config,
|
||||
diffusion_config=diffusion_config,
|
||||
@@ -3223,10 +3109,6 @@ class AsyncEngineArgs(EngineArgs):
|
||||
def add_cli_args(
|
||||
parser: FlexibleArgumentParser, async_args_only: bool = False
|
||||
) -> FlexibleArgumentParser:
|
||||
# Initialize plugin to update the parser, for example, The plugin may
|
||||
# add a new kind of quantization method to --quantization argument or
|
||||
# a new device to --device argument.
|
||||
load_general_plugins()
|
||||
if not async_args_only:
|
||||
parser = EngineArgs.add_cli_args(parser)
|
||||
parser.add_argument(
|
||||
@@ -3234,7 +3116,7 @@ class AsyncEngineArgs(EngineArgs):
|
||||
action=argparse.BooleanOptionalAction,
|
||||
default=AsyncEngineArgs.enable_log_requests,
|
||||
help="Enable logging request information, dependent on log level:\n"
|
||||
"- INFO: Request ID, parameters and LoRA request.\n"
|
||||
"- INFO: Request ID and parameters.\n"
|
||||
"- DEBUG: Prompt inputs (e.g: text, token IDs).\n"
|
||||
"You can set the minimum log level via `VLLM_LOGGING_LEVEL`.\n"
|
||||
"You can also set a separate log level for request JSON body "
|
||||
|
||||
@@ -13,7 +13,6 @@ from vllm.distributed.weight_transfer.base import (
|
||||
WeightTransferUpdateRequest,
|
||||
)
|
||||
from vllm.inputs import EngineInput, PromptType
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.outputs import PoolingRequestOutput, RequestOutput
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.renderers import BaseRenderer
|
||||
@@ -101,7 +100,6 @@ class EngineClient(ABC):
|
||||
request_id: str,
|
||||
*,
|
||||
prompt_text: str | None = None,
|
||||
lora_request: LoRARequest | None = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
priority: int = 0,
|
||||
@@ -120,7 +118,6 @@ class EngineClient(ABC):
|
||||
prompt: PromptType | EngineInput,
|
||||
pooling_params: PoolingParams,
|
||||
request_id: str,
|
||||
lora_request: LoRARequest | None = None,
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
priority: int = 0,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
@@ -218,11 +215,6 @@ class EngineClient(ABC):
|
||||
"""Check whether the engine is sleeping."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
async def add_lora(self, lora_request: LoRARequest) -> bool:
|
||||
"""Load a new LoRA adapter into the engine for future requests."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
async def pause_generation(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user