[Mini-VLLM] Remove LoRA, plugins, speech-to-text and scale-out surfaces

Production runs one static model behind the OpenAI-compatible API. Every
adapter path and non-OpenAI HTTP surface is unreachable there.

- Delete the LoRA stack: vllm/lora, LoRARequest on the engine wire, the
  scheduler, worker, cudagraph and MoE plumbing, LoRA metrics, the lora
  CLI group, and the LoRA tests.
- Drop the plugin loader in vllm/plugins. Platform resolution is now
  CUDA-only and the general/stat-logger plugin hooks are gone.
- Delete the speech-to-text and realtime entrypoints, the scale-out
  render routes, and vllm/config/speech_to_text.py. GenerationTask is
  generate-only.
- Cut DecodeBenchConnector and the disagg-prefill doc and image. The
  Multi, Offloading and Example connectors stay; production uses the
  CPU KV offload tier.
- Delete the ROCm-only fused KDA chunk kernel (2346 lines) with its CMake
  arch gate, declarations and op registrations. The CUDA build never
  defined the guard and no Python caller existed.
- Delete the orphan Mistral and Pixtral chat fixtures and examples/.
- Fix _check_model: removing the LoRA early return made every request
  fail with "The model ... does not exist. Valid aliases: ...".

Verified on an RTX 3090: import sweep reports 1242 modules with 7 known
optional-dependency failures; a live server answers chat, completions,
responses, tokenize and detokenize with zero log errors and 404 for the
removed surfaces; collection yields 8558 tests with the 3 pre-existing
errors; ruff adds no new finding; the incremental _C build succeeds.

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 21:06:19 +07:00
co-authored by Hinata
parent 5f3c7bd8ab
commit 5aa2ac64f8
269 changed files with 417 additions and 29932 deletions
+5 -374
View File
@@ -1,19 +1,15 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import weakref
from bisect import bisect_right
from collections.abc import (
AsyncGenerator,
Callable,
Hashable,
Mapping,
MutableSequence,
Sequence,
)
from contextlib import ExitStack, contextmanager, nullcontext
from dataclasses import dataclass
from operator import attrgetter
from typing import (
TYPE_CHECKING,
@@ -27,11 +23,9 @@ from typing import (
runtime_checkable,
)
import numpy as np
import torch
import torch.nn as nn
from torch import Tensor
from transformers.models.whisper.tokenization_whisper import LANGUAGES
from typing_extensions import TypeIs
from vllm.logger import init_logger
@@ -42,14 +36,10 @@ from vllm.utils.func_utils import supports_kw
if TYPE_CHECKING:
from vllm.config import (
ModelConfig,
SpeechToTextConfig,
SpeechToTextParams,
VllmConfig,
)
from vllm.config.multimodal import VideoPruningMethod
from vllm.inputs import PromptType, TokensPrompt
from vllm.lora.model_manager import LoRAModelManager
from vllm.inputs import TokensPrompt
from vllm.model_executor.layers.fused_moe import MoERunner
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.mamba.mamba_utils import (
@@ -58,11 +48,10 @@ if TYPE_CHECKING:
)
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.models.utils import WeightsMapper
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalKwargsItem
from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.multimodal.registry import _ProcessorFactories
from vllm.sequence import IntermediateTensors
from vllm.tasks import ScoreType
from vllm.tokenizers import TokenizerLike
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, KVCacheSpec
from vllm.v1.worker.encoder_cudagraph_defs import (
@@ -105,23 +94,6 @@ MambaStateShapes: TypeAlias = (
)
@dataclass(frozen=True, slots=True)
class DiarizedTranscriptionSegment:
"""A timestamped, speaker-attributed segment produced by an ASR model."""
start: float
end: float
speaker: str
text: str
class StreamingTranscriptionPostProcessor:
"""Stateful streaming post-processor for transcription deltas."""
def process_delta(self, text_delta: str, finished: bool) -> str:
return text_delta
def _require_is_multimodal(is_multimodal: Tensor | None) -> Tensor:
"""A helper function to be used in the context of
[vllm.model_executor.models.interfaces.SupportsMultiModal.embed_input_ids][]
@@ -206,12 +178,6 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
`multimodal_config.mm_device_do_normalize`.
"""
supports_tower_connector_lora: ClassVar[bool] = False
"""
A flag that indicates whether this model supports
`lora_config.enable_tower_connector_lora`.
"""
requires_raw_input_tokens: ClassVar[bool] = False
"""
A flag that indicates this model processes input id tokens
@@ -432,24 +398,6 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
yield
def get_mm_lora_token_counts(
self,
*,
modality: str,
mm_kwargs: "MultiModalKwargsItem | None",
num_mm_embeds: int,
) -> tuple[int, int | None]:
"""Return ``(tower_tokens, connector_tokens)`` for multimodal LoRA mappings.
MM LoRA uses these counts to build adapter mappings for the tower and
connector forwards. Models with multiple modalities can override this
when each modality has different encoder padding or pooling behavior.
"""
raise NotImplementedError(
f"{type(self).__name__} must implement get_mm_lora_token_counts "
"to support tower/connector LoRA."
)
def _embed_text_input_ids(
self,
input_ids: Tensor,
@@ -459,9 +407,9 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
) -> Tensor:
if is_multimodal is not None and self._has_oov_mm_tokens:
# Force all input IDs to be in vocab; we do this instead of squeezing
# to ensure that any external configuration requiring offset tracking,
# e.g., LoRA, are applied correctly regardless of whether or not
# we have multimodal tokens.
# to ensure that any external configuration requiring offset tracking
# is applied correctly regardless of whether or not we have
# multimodal tokens.
in_vocab_ids = input_ids.masked_fill(
is_multimodal.to(device=input_ids.device, non_blocking=True), 0
)
@@ -712,85 +660,6 @@ def supports_kv_cache_grouping(
return isinstance(model, SupportsKVCacheGrouping)
@runtime_checkable
class SupportsLoRA(Protocol):
"""The interface required for all models that support LoRA."""
supports_lora: ClassVar[Literal[True]] = True
"""
A flag that indicates this model supports LoRA.
Note:
There is no need to redefine this flag if this class is in the
MRO of your model class.
"""
is_3d_moe_weight: ClassVar[bool] = False
is_non_gated_moe: ClassVar[bool] = False
# The `embedding_module` and `embedding_padding_modules`
# are empty by default.
embedding_modules: ClassVar[dict[str, str]] = {}
packed_modules_mapping: dict[str, list[str]] = {}
# Module prefixes to skip during LoRA loading (e.g., ["mtp."] for MTP layers)
lora_skip_prefixes: ClassVar[list[str]] = []
lora_manager: "LoRAModelManager | None"
# We can't use runtime_checkable with ClassVar for issubclass checks
# so we need to treat the class as an instance and use isinstance instead
@runtime_checkable
class _SupportsLoRAType(Protocol):
supports_lora: Literal[True]
packed_modules_mapping: dict[str, list[str]]
embedding_modules: dict[str, str]
@overload
def supports_lora(model: type[object]) -> TypeIs[type[SupportsLoRA]]: ...
@overload
def supports_lora(model: object) -> TypeIs[SupportsLoRA]: ...
def supports_lora(
model: type[object] | object,
) -> TypeIs[type[SupportsLoRA]] | TypeIs[SupportsLoRA]:
result = _supports_lora(model)
if not result:
lora_attrs = (
"packed_modules_mapping",
"embedding_modules",
)
missing_attrs = tuple(attr for attr in lora_attrs if not hasattr(model, attr))
if getattr(model, "supports_lora", False):
if missing_attrs:
logger.warning(
"The model (%s) sets `supports_lora=True`, "
"but is missing LoRA-specific attributes: %s",
model,
missing_attrs,
)
else:
if not missing_attrs:
logger.warning(
"The model (%s) contains all LoRA-specific attributes, "
"but does not set `supports_lora=True`.",
model,
)
return result
def _supports_lora(model: type[object] | object) -> bool:
if isinstance(model, type):
return isinstance(model, _SupportsLoRAType)
return isinstance(model, SupportsLoRA)
class _MakeEmptyIntermediateTensors(Protocol):
def __call__(
self,
@@ -1289,244 +1158,6 @@ class SupportsQuant:
self.quant_config.packed_modules_mapping.update(packed_modules_mapping)
@runtime_checkable
class SupportsRealtime(Protocol):
"""The interface required for all models that support transcription."""
supports_realtime: ClassVar[Literal[True]] = True
realtime_max_tokens: ClassVar[int] = 1
"""Maximum tokens to generate per streaming audio segment.
Override in subclasses based on the model's expected output length."""
@classmethod
def buffer_realtime_audio(
cls,
audio_stream: AsyncGenerator[np.ndarray, None],
input_stream: asyncio.Queue[list[int]],
model_config: "ModelConfig",
) -> AsyncGenerator["PromptType", None]: ...
@overload
def supports_realtime(
model: type[object],
) -> TypeIs[type[SupportsRealtime]]: ...
@overload
def supports_realtime(model: object) -> TypeIs[SupportsRealtime]: ...
def supports_realtime(
model: type[object] | object,
) -> TypeIs[type[SupportsRealtime]] | TypeIs[SupportsRealtime]:
return getattr(model, "supports_realtime", False)
@runtime_checkable
class SupportsTranscription(Protocol):
"""The interface required for all models that support transcription."""
# Mapping from ISO639_1 language codes: language names
supported_languages: ClassVar[Mapping[str, str]]
supports_transcription: ClassVar[Literal[True]] = True
supports_transcription_only: ClassVar[bool] = False
"""
Transcription models can opt out of text generation by setting this to
`True`.
"""
supports_segment_timestamp: ClassVar[bool] = False
"""
Enables the segment timestamp option for supported models by setting this to `True`.
"""
supports_diarized_transcription: ClassVar[bool] = False
"""Enables the ``diarized_json`` response format for the model."""
supports_explicit_language_detection: ClassVar[bool] = False
"""
Transcription models that require an explicit language detection step
(e.g. Whisper needs a separate forward pass to predict the language
token) should set this to ``True`` and implement
:meth:`get_language_detection_prompt` and
:meth:`parse_language_detection_output` and
:meth:`get_language_token_ids`.
"""
no_space_languages: ClassVar[set[str]] = {"ja", "zh"}
"""
Languages that don't need a space between words.
For example, Japanese (ja) and Chinese (zh) don't need a space between words.
"""
def __init_subclass__(cls, **kwargs):
super().__init_subclass__(**kwargs)
# language codes in supported_languages
# that don't exist in the full language map
invalid = set(cls.supported_languages) - set(LANGUAGES.keys())
if invalid:
raise ValueError(
f"{cls.__name__}.supported_languages contains invalid "
f"language codes: {sorted(invalid)}\n. "
f"Valid choices are: {sorted(LANGUAGES.keys())}"
)
@classmethod
def get_generation_prompt(
cls,
stt_params: "SpeechToTextParams",
) -> "PromptType":
"""Get the prompt for the ASR model.
The model has control over the construction, as long as it
returns a valid PromptType."""
...
@classmethod
def get_other_languages(cls) -> Mapping[str, str]:
# other possible language codes from the whisper map
return {k: v for k, v in LANGUAGES.items() if k not in cls.supported_languages}
@classmethod
def validate_language(cls, language: str | None) -> str | None:
"""Ensure the language specified in the transcription request
is a valid ISO 639-1 language code. If the request language is
valid, but not natively supported by the model, trigger a
warning (but not an exception).
"""
if language is None or language in cls.supported_languages:
return language
elif language in cls.get_other_languages():
logger.warning(
"Language %r is not natively supported by %s; "
"results may be less accurate. Supported languages: %r",
language,
cls.__name__,
list(cls.supported_languages.keys()),
)
return language
else:
raise ValueError(
f"Unsupported language: {language!r}. Must be one of "
f"{list(cls.supported_languages.keys())}."
)
@classmethod
def get_speech_to_text_config(
cls, model_config: "ModelConfig", task_type: Literal["transcribe", "translate"]
) -> "SpeechToTextConfig":
"""Get the speech to text config for the ASR model."""
...
@classmethod
def get_num_audio_tokens(
cls,
audio_duration_s: float,
stt_config: "SpeechToTextConfig",
model_config: "ModelConfig",
) -> int | None:
"""Map from audio duration to number of audio tokens produced by the ASR
model, without running a forward pass.
This is used for estimating the amount of processing for this audio.
"""
return None
@classmethod
def post_process_output(cls, text: str) -> str:
"""Post-process the raw model output text.
Some ASR models output structured formats (e.g., language tags,
special tokens) that need to be stripped before returning to the user.
Args:
text: Raw decoded text from the model.
Returns:
Cleaned transcription text.
"""
return text
@classmethod
def parse_diarized_transcript(cls, text: str) -> list[DiarizedTranscriptionSegment]:
"""Parse the model-specific diarized transcript format.
Only models that set ``supports_diarized_transcription`` must override
this method.
"""
raise NotImplementedError
@classmethod
def get_streaming_post_processor_cls(
cls,
) -> type[StreamingTranscriptionPostProcessor]:
"""Return a stateful post-processor class for streaming output deltas.
Each instance receives the next decoded text delta and whether the
request output is final. It returns the cleaned delta that should be
sent to the client.
"""
return StreamingTranscriptionPostProcessor
@classmethod
def get_language_detection_prompt(
cls,
audio: np.ndarray,
stt_config: "SpeechToTextConfig",
) -> "PromptType":
"""Return a prompt that triggers language detection.
Only needs to be implemented when
``supports_explicit_language_detection`` is ``True``.
"""
raise NotImplementedError
@classmethod
def parse_language_detection_output(
cls,
token_ids: list[int],
tokenizer: "TokenizerLike",
) -> str:
"""Parse the detected language from model output token IDs.
Only needs to be implemented when
``supports_explicit_language_detection`` is ``True``.
"""
raise NotImplementedError
@classmethod
def get_language_token_ids(
cls,
tokenizer: "TokenizerLike",
) -> list[int] | None:
"""Return token IDs that represent valid language tokens.
Used to constrain language detection to only produce valid language tokens.
Only needs to be implemented when
``supports_explicit_language_detection`` is ``True``.
"""
raise NotImplementedError
@overload
def supports_transcription(
model: type[object],
) -> TypeIs[type[SupportsTranscription]]: ...
@overload
def supports_transcription(model: object) -> TypeIs[SupportsTranscription]: ...
def supports_transcription(
model: type[object] | object,
) -> TypeIs[type[SupportsTranscription]] | TypeIs[SupportsTranscription]:
return getattr(model, "supports_transcription", False)
@runtime_checkable
class SupportsEagleBase(Protocol):
"""Base interface for models that support EAGLE-based speculative decoding."""