[Mini-VLLM] Remove LoRA, plugins, speech-to-text and scale-out surfaces
Production runs one static model behind the OpenAI-compatible API. Every adapter path and non-OpenAI HTTP surface is unreachable there. - Delete the LoRA stack: vllm/lora, LoRARequest on the engine wire, the scheduler, worker, cudagraph and MoE plumbing, LoRA metrics, the lora CLI group, and the LoRA tests. - Drop the plugin loader in vllm/plugins. Platform resolution is now CUDA-only and the general/stat-logger plugin hooks are gone. - Delete the speech-to-text and realtime entrypoints, the scale-out render routes, and vllm/config/speech_to_text.py. GenerationTask is generate-only. - Cut DecodeBenchConnector and the disagg-prefill doc and image. The Multi, Offloading and Example connectors stay; production uses the CPU KV offload tier. - Delete the ROCm-only fused KDA chunk kernel (2346 lines) with its CMake arch gate, declarations and op registrations. The CUDA build never defined the guard and no Python caller existed. - Delete the orphan Mistral and Pixtral chat fixtures and examples/. - Fix _check_model: removing the LoRA early return made every request fail with "The model ... does not exist. Valid aliases: ...". Verified on an RTX 3090: import sweep reports 1242 modules with 7 known optional-dependency failures; a live server answers chat, completions, responses, tokenize and detokenize with zero log errors and 404 for the removed surfaces; collection yields 8558 tests with the 3 pre-existing errors; ruff adds no new finding; the incremental _C build succeeds. Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -4,20 +4,16 @@
|
||||
from .interfaces import (
|
||||
HasInnerState,
|
||||
SupportsHCSP,
|
||||
SupportsLoRA,
|
||||
SupportsMRoPE,
|
||||
SupportsMultiModal,
|
||||
SupportsMultiModalEmbeddings,
|
||||
SupportsPP,
|
||||
SupportsTranscription,
|
||||
has_inner_state,
|
||||
supports_hc_sp,
|
||||
supports_lora,
|
||||
supports_mrope,
|
||||
supports_multimodal,
|
||||
supports_multimodal_embeddings,
|
||||
supports_pp,
|
||||
supports_transcription,
|
||||
)
|
||||
from .interfaces_base import (
|
||||
VllmModelForPooling,
|
||||
@@ -37,8 +33,6 @@ __all__ = [
|
||||
"has_inner_state",
|
||||
"SupportsHCSP",
|
||||
"supports_hc_sp",
|
||||
"SupportsLoRA",
|
||||
"supports_lora",
|
||||
"SupportsMultiModal",
|
||||
"supports_multimodal",
|
||||
"SupportsMultiModalEmbeddings",
|
||||
@@ -47,6 +41,4 @@ __all__ = [
|
||||
"supports_mrope",
|
||||
"SupportsPP",
|
||||
"supports_pp",
|
||||
"SupportsTranscription",
|
||||
"supports_transcription",
|
||||
]
|
||||
|
||||
@@ -1,19 +1,15 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import weakref
|
||||
from bisect import bisect_right
|
||||
from collections.abc import (
|
||||
AsyncGenerator,
|
||||
Callable,
|
||||
Hashable,
|
||||
Mapping,
|
||||
MutableSequence,
|
||||
Sequence,
|
||||
)
|
||||
from contextlib import ExitStack, contextmanager, nullcontext
|
||||
from dataclasses import dataclass
|
||||
from operator import attrgetter
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
@@ -27,11 +23,9 @@ from typing import (
|
||||
runtime_checkable,
|
||||
)
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch import Tensor
|
||||
from transformers.models.whisper.tokenization_whisper import LANGUAGES
|
||||
from typing_extensions import TypeIs
|
||||
|
||||
from vllm.logger import init_logger
|
||||
@@ -42,14 +36,10 @@ from vllm.utils.func_utils import supports_kw
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import (
|
||||
ModelConfig,
|
||||
SpeechToTextConfig,
|
||||
SpeechToTextParams,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.config.multimodal import VideoPruningMethod
|
||||
from vllm.inputs import PromptType, TokensPrompt
|
||||
from vllm.lora.model_manager import LoRAModelManager
|
||||
from vllm.inputs import TokensPrompt
|
||||
from vllm.model_executor.layers.fused_moe import MoERunner
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
@@ -58,11 +48,10 @@ if TYPE_CHECKING:
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalKwargsItem
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.multimodal.registry import _ProcessorFactories
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tasks import ScoreType
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, KVCacheSpec
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import (
|
||||
@@ -105,23 +94,6 @@ MambaStateShapes: TypeAlias = (
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DiarizedTranscriptionSegment:
|
||||
"""A timestamped, speaker-attributed segment produced by an ASR model."""
|
||||
|
||||
start: float
|
||||
end: float
|
||||
speaker: str
|
||||
text: str
|
||||
|
||||
|
||||
class StreamingTranscriptionPostProcessor:
|
||||
"""Stateful streaming post-processor for transcription deltas."""
|
||||
|
||||
def process_delta(self, text_delta: str, finished: bool) -> str:
|
||||
return text_delta
|
||||
|
||||
|
||||
def _require_is_multimodal(is_multimodal: Tensor | None) -> Tensor:
|
||||
"""A helper function to be used in the context of
|
||||
[vllm.model_executor.models.interfaces.SupportsMultiModal.embed_input_ids][]
|
||||
@@ -206,12 +178,6 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
||||
`multimodal_config.mm_device_do_normalize`.
|
||||
"""
|
||||
|
||||
supports_tower_connector_lora: ClassVar[bool] = False
|
||||
"""
|
||||
A flag that indicates whether this model supports
|
||||
`lora_config.enable_tower_connector_lora`.
|
||||
"""
|
||||
|
||||
requires_raw_input_tokens: ClassVar[bool] = False
|
||||
"""
|
||||
A flag that indicates this model processes input id tokens
|
||||
@@ -432,24 +398,6 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
||||
|
||||
yield
|
||||
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: "MultiModalKwargsItem | None",
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
"""Return ``(tower_tokens, connector_tokens)`` for multimodal LoRA mappings.
|
||||
|
||||
MM LoRA uses these counts to build adapter mappings for the tower and
|
||||
connector forwards. Models with multiple modalities can override this
|
||||
when each modality has different encoder padding or pooling behavior.
|
||||
"""
|
||||
raise NotImplementedError(
|
||||
f"{type(self).__name__} must implement get_mm_lora_token_counts "
|
||||
"to support tower/connector LoRA."
|
||||
)
|
||||
|
||||
def _embed_text_input_ids(
|
||||
self,
|
||||
input_ids: Tensor,
|
||||
@@ -459,9 +407,9 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
||||
) -> Tensor:
|
||||
if is_multimodal is not None and self._has_oov_mm_tokens:
|
||||
# Force all input IDs to be in vocab; we do this instead of squeezing
|
||||
# to ensure that any external configuration requiring offset tracking,
|
||||
# e.g., LoRA, are applied correctly regardless of whether or not
|
||||
# we have multimodal tokens.
|
||||
# to ensure that any external configuration requiring offset tracking
|
||||
# is applied correctly regardless of whether or not we have
|
||||
# multimodal tokens.
|
||||
in_vocab_ids = input_ids.masked_fill(
|
||||
is_multimodal.to(device=input_ids.device, non_blocking=True), 0
|
||||
)
|
||||
@@ -712,85 +660,6 @@ def supports_kv_cache_grouping(
|
||||
return isinstance(model, SupportsKVCacheGrouping)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsLoRA(Protocol):
|
||||
"""The interface required for all models that support LoRA."""
|
||||
|
||||
supports_lora: ClassVar[Literal[True]] = True
|
||||
"""
|
||||
A flag that indicates this model supports LoRA.
|
||||
|
||||
Note:
|
||||
There is no need to redefine this flag if this class is in the
|
||||
MRO of your model class.
|
||||
"""
|
||||
is_3d_moe_weight: ClassVar[bool] = False
|
||||
is_non_gated_moe: ClassVar[bool] = False
|
||||
# The `embedding_module` and `embedding_padding_modules`
|
||||
# are empty by default.
|
||||
embedding_modules: ClassVar[dict[str, str]] = {}
|
||||
packed_modules_mapping: dict[str, list[str]] = {}
|
||||
# Module prefixes to skip during LoRA loading (e.g., ["mtp."] for MTP layers)
|
||||
lora_skip_prefixes: ClassVar[list[str]] = []
|
||||
lora_manager: "LoRAModelManager | None"
|
||||
|
||||
|
||||
# We can't use runtime_checkable with ClassVar for issubclass checks
|
||||
# so we need to treat the class as an instance and use isinstance instead
|
||||
@runtime_checkable
|
||||
class _SupportsLoRAType(Protocol):
|
||||
supports_lora: Literal[True]
|
||||
|
||||
packed_modules_mapping: dict[str, list[str]]
|
||||
embedding_modules: dict[str, str]
|
||||
|
||||
|
||||
@overload
|
||||
def supports_lora(model: type[object]) -> TypeIs[type[SupportsLoRA]]: ...
|
||||
|
||||
|
||||
@overload
|
||||
def supports_lora(model: object) -> TypeIs[SupportsLoRA]: ...
|
||||
|
||||
|
||||
def supports_lora(
|
||||
model: type[object] | object,
|
||||
) -> TypeIs[type[SupportsLoRA]] | TypeIs[SupportsLoRA]:
|
||||
result = _supports_lora(model)
|
||||
|
||||
if not result:
|
||||
lora_attrs = (
|
||||
"packed_modules_mapping",
|
||||
"embedding_modules",
|
||||
)
|
||||
missing_attrs = tuple(attr for attr in lora_attrs if not hasattr(model, attr))
|
||||
|
||||
if getattr(model, "supports_lora", False):
|
||||
if missing_attrs:
|
||||
logger.warning(
|
||||
"The model (%s) sets `supports_lora=True`, "
|
||||
"but is missing LoRA-specific attributes: %s",
|
||||
model,
|
||||
missing_attrs,
|
||||
)
|
||||
else:
|
||||
if not missing_attrs:
|
||||
logger.warning(
|
||||
"The model (%s) contains all LoRA-specific attributes, "
|
||||
"but does not set `supports_lora=True`.",
|
||||
model,
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _supports_lora(model: type[object] | object) -> bool:
|
||||
if isinstance(model, type):
|
||||
return isinstance(model, _SupportsLoRAType)
|
||||
|
||||
return isinstance(model, SupportsLoRA)
|
||||
|
||||
|
||||
class _MakeEmptyIntermediateTensors(Protocol):
|
||||
def __call__(
|
||||
self,
|
||||
@@ -1289,244 +1158,6 @@ class SupportsQuant:
|
||||
self.quant_config.packed_modules_mapping.update(packed_modules_mapping)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsRealtime(Protocol):
|
||||
"""The interface required for all models that support transcription."""
|
||||
|
||||
supports_realtime: ClassVar[Literal[True]] = True
|
||||
|
||||
realtime_max_tokens: ClassVar[int] = 1
|
||||
"""Maximum tokens to generate per streaming audio segment.
|
||||
Override in subclasses based on the model's expected output length."""
|
||||
|
||||
@classmethod
|
||||
def buffer_realtime_audio(
|
||||
cls,
|
||||
audio_stream: AsyncGenerator[np.ndarray, None],
|
||||
input_stream: asyncio.Queue[list[int]],
|
||||
model_config: "ModelConfig",
|
||||
) -> AsyncGenerator["PromptType", None]: ...
|
||||
|
||||
|
||||
@overload
|
||||
def supports_realtime(
|
||||
model: type[object],
|
||||
) -> TypeIs[type[SupportsRealtime]]: ...
|
||||
|
||||
|
||||
@overload
|
||||
def supports_realtime(model: object) -> TypeIs[SupportsRealtime]: ...
|
||||
|
||||
|
||||
def supports_realtime(
|
||||
model: type[object] | object,
|
||||
) -> TypeIs[type[SupportsRealtime]] | TypeIs[SupportsRealtime]:
|
||||
return getattr(model, "supports_realtime", False)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsTranscription(Protocol):
|
||||
"""The interface required for all models that support transcription."""
|
||||
|
||||
# Mapping from ISO639_1 language codes: language names
|
||||
supported_languages: ClassVar[Mapping[str, str]]
|
||||
|
||||
supports_transcription: ClassVar[Literal[True]] = True
|
||||
|
||||
supports_transcription_only: ClassVar[bool] = False
|
||||
"""
|
||||
Transcription models can opt out of text generation by setting this to
|
||||
`True`.
|
||||
"""
|
||||
supports_segment_timestamp: ClassVar[bool] = False
|
||||
"""
|
||||
Enables the segment timestamp option for supported models by setting this to `True`.
|
||||
"""
|
||||
|
||||
supports_diarized_transcription: ClassVar[bool] = False
|
||||
"""Enables the ``diarized_json`` response format for the model."""
|
||||
|
||||
supports_explicit_language_detection: ClassVar[bool] = False
|
||||
"""
|
||||
Transcription models that require an explicit language detection step
|
||||
(e.g. Whisper needs a separate forward pass to predict the language
|
||||
token) should set this to ``True`` and implement
|
||||
:meth:`get_language_detection_prompt` and
|
||||
:meth:`parse_language_detection_output` and
|
||||
:meth:`get_language_token_ids`.
|
||||
"""
|
||||
|
||||
no_space_languages: ClassVar[set[str]] = {"ja", "zh"}
|
||||
"""
|
||||
Languages that don't need a space between words.
|
||||
For example, Japanese (ja) and Chinese (zh) don't need a space between words.
|
||||
"""
|
||||
|
||||
def __init_subclass__(cls, **kwargs):
|
||||
super().__init_subclass__(**kwargs)
|
||||
# language codes in supported_languages
|
||||
# that don't exist in the full language map
|
||||
invalid = set(cls.supported_languages) - set(LANGUAGES.keys())
|
||||
if invalid:
|
||||
raise ValueError(
|
||||
f"{cls.__name__}.supported_languages contains invalid "
|
||||
f"language codes: {sorted(invalid)}\n. "
|
||||
f"Valid choices are: {sorted(LANGUAGES.keys())}"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_generation_prompt(
|
||||
cls,
|
||||
stt_params: "SpeechToTextParams",
|
||||
) -> "PromptType":
|
||||
"""Get the prompt for the ASR model.
|
||||
The model has control over the construction, as long as it
|
||||
returns a valid PromptType."""
|
||||
...
|
||||
|
||||
@classmethod
|
||||
def get_other_languages(cls) -> Mapping[str, str]:
|
||||
# other possible language codes from the whisper map
|
||||
return {k: v for k, v in LANGUAGES.items() if k not in cls.supported_languages}
|
||||
|
||||
@classmethod
|
||||
def validate_language(cls, language: str | None) -> str | None:
|
||||
"""Ensure the language specified in the transcription request
|
||||
is a valid ISO 639-1 language code. If the request language is
|
||||
valid, but not natively supported by the model, trigger a
|
||||
warning (but not an exception).
|
||||
"""
|
||||
if language is None or language in cls.supported_languages:
|
||||
return language
|
||||
elif language in cls.get_other_languages():
|
||||
logger.warning(
|
||||
"Language %r is not natively supported by %s; "
|
||||
"results may be less accurate. Supported languages: %r",
|
||||
language,
|
||||
cls.__name__,
|
||||
list(cls.supported_languages.keys()),
|
||||
)
|
||||
return language
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported language: {language!r}. Must be one of "
|
||||
f"{list(cls.supported_languages.keys())}."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_speech_to_text_config(
|
||||
cls, model_config: "ModelConfig", task_type: Literal["transcribe", "translate"]
|
||||
) -> "SpeechToTextConfig":
|
||||
"""Get the speech to text config for the ASR model."""
|
||||
...
|
||||
|
||||
@classmethod
|
||||
def get_num_audio_tokens(
|
||||
cls,
|
||||
audio_duration_s: float,
|
||||
stt_config: "SpeechToTextConfig",
|
||||
model_config: "ModelConfig",
|
||||
) -> int | None:
|
||||
"""Map from audio duration to number of audio tokens produced by the ASR
|
||||
model, without running a forward pass.
|
||||
This is used for estimating the amount of processing for this audio.
|
||||
"""
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def post_process_output(cls, text: str) -> str:
|
||||
"""Post-process the raw model output text.
|
||||
|
||||
Some ASR models output structured formats (e.g., language tags,
|
||||
special tokens) that need to be stripped before returning to the user.
|
||||
|
||||
Args:
|
||||
text: Raw decoded text from the model.
|
||||
|
||||
Returns:
|
||||
Cleaned transcription text.
|
||||
|
||||
"""
|
||||
return text
|
||||
|
||||
@classmethod
|
||||
def parse_diarized_transcript(cls, text: str) -> list[DiarizedTranscriptionSegment]:
|
||||
"""Parse the model-specific diarized transcript format.
|
||||
|
||||
Only models that set ``supports_diarized_transcription`` must override
|
||||
this method.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def get_streaming_post_processor_cls(
|
||||
cls,
|
||||
) -> type[StreamingTranscriptionPostProcessor]:
|
||||
"""Return a stateful post-processor class for streaming output deltas.
|
||||
|
||||
Each instance receives the next decoded text delta and whether the
|
||||
request output is final. It returns the cleaned delta that should be
|
||||
sent to the client.
|
||||
"""
|
||||
return StreamingTranscriptionPostProcessor
|
||||
|
||||
@classmethod
|
||||
def get_language_detection_prompt(
|
||||
cls,
|
||||
audio: np.ndarray,
|
||||
stt_config: "SpeechToTextConfig",
|
||||
) -> "PromptType":
|
||||
"""Return a prompt that triggers language detection.
|
||||
|
||||
Only needs to be implemented when
|
||||
``supports_explicit_language_detection`` is ``True``.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def parse_language_detection_output(
|
||||
cls,
|
||||
token_ids: list[int],
|
||||
tokenizer: "TokenizerLike",
|
||||
) -> str:
|
||||
"""Parse the detected language from model output token IDs.
|
||||
|
||||
Only needs to be implemented when
|
||||
``supports_explicit_language_detection`` is ``True``.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def get_language_token_ids(
|
||||
cls,
|
||||
tokenizer: "TokenizerLike",
|
||||
) -> list[int] | None:
|
||||
"""Return token IDs that represent valid language tokens.
|
||||
|
||||
Used to constrain language detection to only produce valid language tokens.
|
||||
|
||||
Only needs to be implemented when
|
||||
``supports_explicit_language_detection`` is ``True``.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@overload
|
||||
def supports_transcription(
|
||||
model: type[object],
|
||||
) -> TypeIs[type[SupportsTranscription]]: ...
|
||||
|
||||
|
||||
@overload
|
||||
def supports_transcription(model: object) -> TypeIs[SupportsTranscription]: ...
|
||||
|
||||
|
||||
def supports_transcription(
|
||||
model: type[object] | object,
|
||||
) -> TypeIs[type[SupportsTranscription]] | TypeIs[SupportsTranscription]:
|
||||
return getattr(model, "supports_transcription", False)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsEagleBase(Protocol):
|
||||
"""Base interface for models that support EAGLE-based speculative decoding."""
|
||||
|
||||
@@ -62,7 +62,6 @@ from .interfaces import (
|
||||
EagleModelMixin,
|
||||
SupportsEagle,
|
||||
SupportsEagle3,
|
||||
SupportsLoRA,
|
||||
SupportsPP,
|
||||
SupportsQuant,
|
||||
)
|
||||
@@ -456,7 +455,7 @@ class Qwen2Model(nn.Module, EagleModelMixin):
|
||||
|
||||
|
||||
class Qwen2ForCausalLM(
|
||||
nn.Module, SupportsLoRA, SupportsPP, SupportsEagle, SupportsEagle3, SupportsQuant
|
||||
nn.Module, SupportsPP, SupportsEagle, SupportsEagle3, SupportsQuant
|
||||
):
|
||||
hf_to_vllm_mapper = Qwen2Model.hf_to_vllm_mapper
|
||||
packed_modules_mapping = {
|
||||
|
||||
@@ -76,7 +76,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
VideoItem,
|
||||
)
|
||||
@@ -103,7 +102,6 @@ from .interfaces import (
|
||||
SupportsEagle,
|
||||
SupportsEagle3,
|
||||
SupportsEncoderCudaGraph,
|
||||
SupportsLoRA,
|
||||
SupportsMRoPE,
|
||||
SupportsMultiModal,
|
||||
SupportsMultiModalPruning,
|
||||
@@ -1295,7 +1293,6 @@ class Qwen2_5_VLForConditionalGeneration(
|
||||
nn.Module,
|
||||
SupportsMultiModal,
|
||||
SupportsEncoderCudaGraph,
|
||||
SupportsLoRA,
|
||||
SupportsPP,
|
||||
SupportsQuant,
|
||||
SupportsEagle,
|
||||
@@ -1323,7 +1320,6 @@ class Qwen2_5_VLForConditionalGeneration(
|
||||
|
||||
supports_encoder_tp_data = True
|
||||
supports_mm_device_do_normalize = True
|
||||
supports_tower_connector_lora = True
|
||||
|
||||
def iter_mm_grid_thw(
|
||||
self, mm_features: list[MultiModalFeatureSpec]
|
||||
@@ -2129,16 +2125,3 @@ class Qwen2_5_VLForConditionalGeneration(
|
||||
connector="visual.merger.",
|
||||
tower_model="visual.",
|
||||
)
|
||||
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
hf_config = self.config
|
||||
vision_config = hf_config.vision_config
|
||||
merge_size = vision_config.spatial_merge_size
|
||||
return num_mm_embeds * merge_size**2, num_mm_embeds
|
||||
|
||||
@@ -60,7 +60,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import SupportsLoRA, SupportsPP
|
||||
from .interfaces import SupportsPP
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
@@ -420,7 +420,7 @@ class Qwen2MoeModel(nn.Module):
|
||||
return hidden_states
|
||||
|
||||
|
||||
class Qwen2MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
|
||||
class Qwen2MoeForCausalLM(nn.Module, SupportsPP):
|
||||
fall_back_to_pt_during_load = False
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
|
||||
@@ -70,7 +70,6 @@ from vllm.multimodal.inputs import (
|
||||
ImageItem,
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
VideoItem,
|
||||
)
|
||||
@@ -97,7 +96,6 @@ from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers
|
||||
from .interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsEncoderCudaGraph,
|
||||
SupportsLoRA,
|
||||
SupportsMRoPE,
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
@@ -1286,7 +1284,6 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo])
|
||||
class Qwen2VLForConditionalGeneration(
|
||||
nn.Module,
|
||||
SupportsMultiModal,
|
||||
SupportsLoRA,
|
||||
SupportsPP,
|
||||
SupportsMRoPE,
|
||||
SupportsEncoderCudaGraph,
|
||||
@@ -1305,7 +1302,6 @@ class Qwen2VLForConditionalGeneration(
|
||||
|
||||
supports_encoder_tp_data = True
|
||||
supports_mm_device_do_normalize = True
|
||||
supports_tower_connector_lora = True
|
||||
|
||||
def iter_mm_grid_thw(
|
||||
self, mm_features: list[MultiModalFeatureSpec]
|
||||
@@ -1860,16 +1856,3 @@ class Qwen2VLForConditionalGeneration(
|
||||
connector="visual.merger.",
|
||||
tower_model="visual.",
|
||||
)
|
||||
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
hf_config = self.config
|
||||
vision_config = hf_config.vision_config
|
||||
merge_size = vision_config.spatial_merge_size
|
||||
return num_mm_embeds * merge_size**2, num_mm_embeds
|
||||
|
||||
@@ -52,7 +52,6 @@ from .interfaces import (
|
||||
LocalArgmaxMixin,
|
||||
SupportsEagle,
|
||||
SupportsEagle3,
|
||||
SupportsLoRA,
|
||||
SupportsPP,
|
||||
)
|
||||
from .qwen2 import Qwen2MLP as Qwen3MLP
|
||||
@@ -278,17 +277,13 @@ class Qwen3Model(Qwen2Model):
|
||||
|
||||
|
||||
class Qwen3ForCausalLM(
|
||||
LocalArgmaxMixin, nn.Module, SupportsLoRA, SupportsPP, SupportsEagle, SupportsEagle3
|
||||
LocalArgmaxMixin, nn.Module, SupportsPP, SupportsEagle, SupportsEagle3
|
||||
):
|
||||
hf_to_vllm_mapper = Qwen3Model.hf_to_vllm_mapper
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||
}
|
||||
embedding_modules = {
|
||||
"embed_tokens": "input_embeddings",
|
||||
"lm_head": "output_embeddings",
|
||||
}
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
@@ -25,7 +25,6 @@
|
||||
"""Inference-only Qwen3.5 Series compatible with HuggingFace weights."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import ClassVar
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
@@ -76,7 +75,6 @@ from .interfaces import (
|
||||
MixtureOfExperts,
|
||||
MultiModalEmbeddings,
|
||||
SupportsEagle3,
|
||||
SupportsLoRA,
|
||||
SupportsMRoPE,
|
||||
SupportsPP,
|
||||
_require_is_multimodal,
|
||||
@@ -328,7 +326,6 @@ class Qwen3_5ForCausalLMBase(
|
||||
HasInnerState,
|
||||
IsHybrid,
|
||||
SupportsEagle3,
|
||||
SupportsLoRA,
|
||||
SupportsMRoPE,
|
||||
SupportsPP,
|
||||
):
|
||||
@@ -343,11 +340,6 @@ class Qwen3_5ForCausalLMBase(
|
||||
"in_proj_qkvz": ["in_proj_qkv", "in_proj_z"],
|
||||
"in_proj_ba": ["in_proj_b", "in_proj_a"],
|
||||
}
|
||||
# Maps PEFT embed/lm_head LoRA targets onto vLLM embedding wrappers.
|
||||
embedding_modules = {
|
||||
"embed_tokens": "input_embeddings",
|
||||
"lm_head": "output_embeddings",
|
||||
}
|
||||
|
||||
# Some community text-only checkpoints keep the extraneous
|
||||
# `model.language_model.` prefix inherited from the VL training stack.
|
||||
@@ -731,9 +723,6 @@ class Qwen3_5_MoeMixtureOfExperts(MixtureOfExperts):
|
||||
class Qwen3_5MoeForConditionalGeneration(
|
||||
Qwen3_5ForConditionalGeneration, Qwen3_5_MoeMixtureOfExperts
|
||||
):
|
||||
# For MoE LoRA weights loading
|
||||
is_3d_moe_weight: ClassVar[bool] = True
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "model"):
|
||||
# protocols have not __init__ method, so we need to use nn.Module.__init__
|
||||
nn.Module.__init__(self)
|
||||
|
||||
@@ -68,7 +68,6 @@ from .interfaces import (
|
||||
IsHybrid,
|
||||
MixtureOfExperts,
|
||||
SupportsEagle3,
|
||||
SupportsLoRA,
|
||||
SupportsPP,
|
||||
)
|
||||
from .utils import (
|
||||
@@ -820,7 +819,6 @@ class QwenNextMixtureOfExperts(MixtureOfExperts):
|
||||
class Qwen3NextForCausalLM(
|
||||
nn.Module,
|
||||
HasInnerState,
|
||||
SupportsLoRA,
|
||||
SupportsPP,
|
||||
QwenNextMixtureOfExperts,
|
||||
IsHybrid,
|
||||
|
||||
@@ -129,7 +129,6 @@ from .interfaces import (
|
||||
SupportsEagle,
|
||||
SupportsEagle3,
|
||||
SupportsEncoderCudaGraph,
|
||||
SupportsLoRA,
|
||||
SupportsMRoPE,
|
||||
SupportsMultiModal,
|
||||
SupportsMultiModalPruning,
|
||||
@@ -1961,7 +1960,6 @@ class Qwen3VLForConditionalGeneration(
|
||||
nn.Module,
|
||||
SupportsMultiModal,
|
||||
SupportsEncoderCudaGraph,
|
||||
SupportsLoRA,
|
||||
SupportsPP,
|
||||
SupportsMRoPE,
|
||||
SupportsEagle,
|
||||
@@ -1982,7 +1980,6 @@ class Qwen3VLForConditionalGeneration(
|
||||
}
|
||||
|
||||
supports_encoder_tp_data = True
|
||||
supports_tower_connector_lora = True
|
||||
supports_mm_device_do_normalize = True
|
||||
|
||||
supported_video_pruning_methods = ("evs", "vidcom2")
|
||||
@@ -3258,19 +3255,6 @@ class Qwen3VLForConditionalGeneration(
|
||||
tower_model="visual.",
|
||||
)
|
||||
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
hf_config = self.config
|
||||
vision_config = hf_config.vision_config
|
||||
merge_size = vision_config.spatial_merge_size
|
||||
return num_mm_embeds * merge_size**2, num_mm_embeds
|
||||
|
||||
|
||||
@lru_cache
|
||||
def _cached_tensor(x, device) -> torch.Tensor:
|
||||
|
||||
@@ -55,7 +55,6 @@ from .interfaces import (
|
||||
supports_multimodal_raw_input_only,
|
||||
supports_pp,
|
||||
supports_replayssm,
|
||||
supports_transcription,
|
||||
)
|
||||
from .interfaces_base import (
|
||||
get_attn_type,
|
||||
@@ -232,8 +231,6 @@ class _ModelInfo:
|
||||
is_hybrid: bool
|
||||
has_noops: bool
|
||||
supports_replayssm: bool
|
||||
supports_transcription: bool
|
||||
supports_transcription_only: bool
|
||||
supports_kv_cache_grouping: bool
|
||||
supported_video_pruning_methods: tuple[str, ...]
|
||||
supports_mm_device_do_normalize: bool
|
||||
@@ -262,10 +259,6 @@ class _ModelInfo:
|
||||
is_attention_free=is_attention_free(model),
|
||||
is_hybrid=is_hybrid(model),
|
||||
supports_replayssm=supports_replayssm(model),
|
||||
supports_transcription=supports_transcription(model),
|
||||
supports_transcription_only=(
|
||||
supports_transcription(model) and model.supports_transcription_only
|
||||
),
|
||||
supports_kv_cache_grouping=supports_kv_cache_grouping(model),
|
||||
has_noops=has_noops(model),
|
||||
supported_video_pruning_methods=getattr(
|
||||
@@ -859,22 +852,6 @@ class _ModelRegistry:
|
||||
model_info, _ = self.inspect_model_cls(architectures, model_config)
|
||||
return model_info.has_noops
|
||||
|
||||
def is_transcription_model(
|
||||
self,
|
||||
architectures: str | list[str],
|
||||
model_config: ModelConfig,
|
||||
) -> bool:
|
||||
model_info, _ = self.inspect_model_cls(architectures, model_config)
|
||||
return model_info.supports_transcription
|
||||
|
||||
def is_transcription_only_model(
|
||||
self,
|
||||
architectures: str | list[str],
|
||||
model_config: ModelConfig,
|
||||
) -> bool:
|
||||
model_info, _ = self.inspect_model_cls(architectures, model_config)
|
||||
return model_info.supports_transcription_only
|
||||
|
||||
|
||||
def _resolve_module_name(mod_relname: str) -> str:
|
||||
# Allow registry entries to point at fully-qualified module paths (e.g.
|
||||
@@ -929,11 +906,6 @@ def _run_in_subprocess(fn: Callable[[], _T]) -> _T:
|
||||
|
||||
|
||||
def _run() -> None:
|
||||
# Setup plugins
|
||||
from vllm.plugins import load_general_plugins
|
||||
|
||||
load_general_plugins()
|
||||
|
||||
fn, output_file = pickle.loads(sys.stdin.buffer.read())
|
||||
|
||||
result = fn()
|
||||
|
||||
@@ -181,13 +181,13 @@ class WeightsMapper:
|
||||
"""Mapper variant keeping only the renames.
|
||||
|
||||
This is what consumers that *name* modules rather than load them need:
|
||||
LoRA name parsing and the quantization config's layer lists.
|
||||
the quantization config's layer lists.
|
||||
|
||||
Stacked maps are dropped so that constituent names (e.g. `q_proj`) survive
|
||||
rather than being rewritten to the stacked vLLM name (`qkv_proj`). Mappings to
|
||||
`None` are dropped because "do not load this weight" is meaningless to such a
|
||||
consumer, and applying it would silently shrink a quantization config's ignore
|
||||
list or make LoRA name parsing fail."""
|
||||
consumer, and applying it would silently shrink a quantization config's
|
||||
ignore list."""
|
||||
remove_none = lambda d: {k: v for k, v in d.items() if v is not None}
|
||||
return replace(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user