Files
MOVZXandHinata 5aa2ac64f8 [Mini-VLLM] Remove LoRA, plugins, speech-to-text and scale-out surfaces
Production runs one static model behind the OpenAI-compatible API. Every
adapter path and non-OpenAI HTTP surface is unreachable there.

- Delete the LoRA stack: vllm/lora, LoRARequest on the engine wire, the
  scheduler, worker, cudagraph and MoE plumbing, LoRA metrics, the lora
  CLI group, and the LoRA tests.
- Drop the plugin loader in vllm/plugins. Platform resolution is now
  CUDA-only and the general/stat-logger plugin hooks are gone.
- Delete the speech-to-text and realtime entrypoints, the scale-out
  render routes, and vllm/config/speech_to_text.py. GenerationTask is
  generate-only.
- Cut DecodeBenchConnector and the disagg-prefill doc and image. The
  Multi, Offloading and Example connectors stay; production uses the
  CPU KV offload tier.
- Delete the ROCm-only fused KDA chunk kernel (2346 lines) with its CMake
  arch gate, declarations and op registrations. The CUDA build never
  defined the guard and no Python caller existed.
- Delete the orphan Mistral and Pixtral chat fixtures and examples/.
- Fix _check_model: removing the LoRA early return made every request
  fail with "The model ... does not exist. Valid aliases: ...".

Verified on an RTX 3090: import sweep reports 1242 modules with 7 known
optional-dependency failures; a live server answers chat, completions,
responses, tokenize and detokenize with zero log errors and 404 for the
removed surfaces; collection yields 8558 tests with the 3 pre-existing
errors; ruff adds no new finding; the incremental _C build succeeds.

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
2026-10-19 21:06:19 +07:00

269 lines
9.9 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import contextlib
from dataclasses import dataclass, field
from functools import lru_cache
from pathlib import Path
from typing import TYPE_CHECKING, assert_never, cast
import huggingface_hub
from typing_extensions import TypeVar
from vllm import envs
from vllm.logger import init_logger
from vllm.transformers_utils.config import _maybe_register_hf_config, get_config
from vllm.transformers_utils.repo_utils import get_non_weight_snapshot_path
from vllm.utils.import_utils import resolve_obj_by_qualname
from .hf import CachedHfTokenizer, HfTokenizer
from .protocol import TokenizerLike
if TYPE_CHECKING:
from vllm.config.model import ModelConfig, RunnerType
logger = init_logger(__name__)
# Model types whose hub tokenizer_class is incorrect and should be overridden with
# TokenizersBackend (the generic fast tokenizer). Adding a model type here is always a
# temporary workaround and better long term solutions are:
# - Add model type to MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS in transformers (better)
# - Fix tokenizer_class on the hub for the affected models (best)
_MODEL_TYPES_WITH_INCORRECT_TOKENIZER_CLASS: set[str] = {
"internlm2",
"step3_vl",
"step3p7",
"unlimited-ocr",
"unlimited_ocr",
}
_VLLM_TOKENIZERS = {
"hf": ("hf", "CachedHfTokenizer"),
}
@dataclass
class _TokenizerRegistry:
# Tokenizer mode -> (tokenizer module, tokenizer class)
tokenizers: dict[str, tuple[str, str]] = field(default_factory=dict)
def register(self, tokenizer_mode: str, module: str, class_name: str) -> None:
if tokenizer_mode in self.tokenizers:
logger.warning(
"%s.%s is already registered for tokenizer_mode=%r. "
"It is overwritten by the new one.",
module,
class_name,
tokenizer_mode,
)
self.tokenizers[tokenizer_mode] = (module, class_name)
def load_tokenizer_cls(self, tokenizer_mode: str) -> type[TokenizerLike]:
if tokenizer_mode not in self.tokenizers:
raise ValueError(f"No tokenizer registered for {tokenizer_mode=!r}.")
module, class_name = self.tokenizers[tokenizer_mode]
logger.debug_once(f"Loading {class_name} for {tokenizer_mode=!r}")
return resolve_obj_by_qualname(f"{module}.{class_name}")
def load_tokenizer(self, tokenizer_mode: str, *args, **kwargs) -> TokenizerLike:
tokenizer_cls = self.load_tokenizer_cls(tokenizer_mode)
return tokenizer_cls.from_pretrained(*args, **kwargs)
TokenizerRegistry = _TokenizerRegistry(
{
mode: (f"vllm.tokenizers.{mod_relname}", cls_name)
for mode, (mod_relname, cls_name) in _VLLM_TOKENIZERS.items()
}
)
def resolve_tokenizer_args(
tokenizer_name: str | Path,
*args,
runner_type: "RunnerType" = "generate",
tokenizer_mode: str = "auto",
**kwargs,
):
revision: str | None = kwargs.get("revision")
download_dir: str | None = kwargs.get("download_dir")
if envs.VLLM_USE_MODELSCOPE:
# download model from ModelScope hub,
# lazy import so that modelscope is not required for normal use.
from modelscope.hub.snapshot_download import snapshot_download
# avoid circular import
from vllm.model_executor.model_loader.weight_utils import get_lock
# Only set the tokenizer here, model will be downloaded on the workers.
if not Path(tokenizer_name).exists():
# Use file lock to prevent multiple processes from
# downloading the same file at the same time.
with get_lock(tokenizer_name, download_dir):
tokenizer_path = snapshot_download(
model_id=str(tokenizer_name),
cache_dir=download_dir,
revision=revision,
local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
# Ignore weights - we only need the tokenizer.
ignore_file_pattern=[".*.pt", ".*.safetensors", ".*.bin"],
)
tokenizer_name = tokenizer_path
if "truncation_side" not in kwargs:
if runner_type == "generate" or runner_type == "draft":
kwargs["truncation_side"] = "left"
elif runner_type == "pooling":
kwargs["truncation_side"] = "right"
else:
assert_never(runner_type)
# Fallback to HF tokenizer
if tokenizer_mode == "auto":
tokenizer_mode = "hf"
if tokenizer_mode == "hf" and kwargs.get("code_revision") in (None, revision):
tokenizer_name = get_non_weight_snapshot_path(
str(tokenizer_name),
revision=revision,
cache_dir=download_dir,
token=kwargs.get("token"),
)
return tokenizer_mode, tokenizer_name, args, kwargs
cached_resolve_tokenizer_args = lru_cache(resolve_tokenizer_args)
def tokenizer_args_from_config(config: "ModelConfig", **kwargs):
if config.tokenizer == config.model:
code_revision = getattr(config, "code_revision", None)
if code_revision is not None:
kwargs.setdefault("code_revision", code_revision)
return cached_resolve_tokenizer_args(
config.tokenizer,
runner_type=config.runner_type,
tokenizer_mode=config.tokenizer_mode,
revision=config.tokenizer_revision,
trust_remote_code=config.trust_remote_code,
**kwargs,
)
_T = TypeVar("_T", bound=TokenizerLike, default=TokenizerLike)
def get_tokenizer(
tokenizer_name: str | Path,
*args,
tokenizer_cls: type[_T] = TokenizerLike, # type: ignore[assignment]
trust_remote_code: bool = False,
revision: str | None = None,
download_dir: str | None = None,
**kwargs,
) -> _T:
"""Gets a tokenizer for the given model name via HuggingFace or ModelScope."""
if envs.VLLM_USE_FASTOKENS:
# Process-global, idempotent patch that swaps the Rust BPE backend
# of any HF fast tokenizer loaded afterwards. No-op for non-HF modes.
from .fastokens import apply_fastokens_patch
apply_fastokens_patch()
requested_name = str(tokenizer_name)
tokenizer_mode, tokenizer_name, args, kwargs = cached_resolve_tokenizer_args(
tokenizer_name,
*args,
trust_remote_code=trust_remote_code,
revision=revision,
download_dir=download_dir,
**kwargs,
)
if tokenizer_cls == TokenizerLike:
tokenizer_cls_ = TokenizerRegistry.load_tokenizer_cls(tokenizer_mode)
else:
tokenizer_cls_ = tokenizer_cls
# Ensure that, if the config were to come from vllm.transformers_utils.config, it is
# registered with AutoConfig before the tokenizer is loaded. This is necessary since
# tokenizer_cls_.from_pretrained will call AutoConfig.from_pretrained internally.
# This may fail for paths that don't have a model config, which is
# fine — those don't need custom config registration.
# HF-backed tokenizers must receive the HF config. In a dual-format Mistral
# repository, auto detection intentionally prefers params.json, but passing
# that generic config to AutoTokenizer can select the wrong tokenizer class.
config_format = "hf" if tokenizer_cls_ is CachedHfTokenizer else "auto"
config = None
with contextlib.suppress(ValueError, OSError):
config = get_config(
tokenizer_name,
trust_remote_code=trust_remote_code,
revision=revision,
code_revision=kwargs.get("code_revision"),
config_format=config_format,
token=kwargs.get("token"),
)
# Some models have an incorrect tokenizer_class on the hub.
# For these model types, bypass AutoTokenizer and use TokenizersBackend directly.
model_type = getattr(config, "model_type", None) if config else None
if model_type in _MODEL_TYPES_WITH_INCORRECT_TOKENIZER_CLASS:
from transformers.tokenization_utils_tokenizers import TokenizersBackend
logger.debug(
"Overriding tokenizer_class to TokenizersBackend for model_type=%r",
model_type,
)
tokenizer_cls_ = TokenizersBackend
if config is not None and tokenizer_cls_ is CachedHfTokenizer:
# AutoTokenizer otherwise reloads config.json internally. Reuse the
# config that get_config just loaded successfully so a concurrent Hub
# cache refresh cannot invalidate the file between the two reads.
kwargs.setdefault("config", config)
tokenizer = tokenizer_cls_.from_pretrained(tokenizer_name, *args, **kwargs)
# Report the Hub ID rather than the local snapshot it was loaded from
if tokenizer_mode == "hf" and not envs.VLLM_USE_MODELSCOPE:
cast(HfTokenizer, tokenizer).name_or_path = requested_name
if model_type in _MODEL_TYPES_WITH_INCORRECT_TOKENIZER_CLASS:
from vllm.tokenizers.hf import get_cached_tokenizer
tokenizer = get_cached_tokenizer(tokenizer)
if not tokenizer.is_fast:
logger.warning(
"Using a slow tokenizer. This might cause a significant "
"slowdown. Consider using a fast tokenizer instead."
)
return tokenizer # type: ignore
cached_get_tokenizer = lru_cache(get_tokenizer)
def cached_tokenizer_from_config(model_config: "ModelConfig", **kwargs):
if model_config.skip_tokenizer_init:
return None
_maybe_register_hf_config(getattr(model_config, "hf_config", None))
if model_config.tokenizer == model_config.model:
code_revision = getattr(model_config, "code_revision", None)
if code_revision is not None:
kwargs.setdefault("code_revision", code_revision)
return cached_get_tokenizer(
model_config.tokenizer,
runner_type=model_config.runner_type,
tokenizer_mode=model_config.tokenizer_mode,
revision=model_config.tokenizer_revision,
trust_remote_code=model_config.trust_remote_code,
**kwargs,
)