Files
MOVZXandHinata 5aa2ac64f8 [Mini-VLLM] Remove LoRA, plugins, speech-to-text and scale-out surfaces
Production runs one static model behind the OpenAI-compatible API. Every
adapter path and non-OpenAI HTTP surface is unreachable there.

- Delete the LoRA stack: vllm/lora, LoRARequest on the engine wire, the
  scheduler, worker, cudagraph and MoE plumbing, LoRA metrics, the lora
  CLI group, and the LoRA tests.
- Drop the plugin loader in vllm/plugins. Platform resolution is now
  CUDA-only and the general/stat-logger plugin hooks are gone.
- Delete the speech-to-text and realtime entrypoints, the scale-out
  render routes, and vllm/config/speech_to_text.py. GenerationTask is
  generate-only.
- Cut DecodeBenchConnector and the disagg-prefill doc and image. The
  Multi, Offloading and Example connectors stay; production uses the
  CPU KV offload tier.
- Delete the ROCm-only fused KDA chunk kernel (2346 lines) with its CMake
  arch gate, declarations and op registrations. The CUDA build never
  defined the guard and no Python caller existed.
- Delete the orphan Mistral and Pixtral chat fixtures and examples/.
- Fix _check_model: removing the LoRA early return made every request
  fail with "The model ... does not exist. Valid aliases: ...".

Verified on an RTX 3090: import sweep reports 1242 modules with 7 known
optional-dependency failures; a live server answers chat, completions,
responses, tokenize and detokenize with zero log errors and 404 for the
removed surfaces; collection yields 8558 tests with the 3 pre-existing
errors; ruff adds no new finding; the incremental _C build succeeds.

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
2026-10-19 21:06:19 +07:00

117 lines
4.5 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from collections.abc import Iterable
from typing import cast
import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.model_loader import get_model_loader
from .layerwise import finalize_layerwise_reload, initialize_layerwise_reload
logger = init_logger(__name__)
__all__ = ["get_parameter_for_reload", "reload_weights"]
def get_parameter_for_reload(model: nn.Module, name: str) -> nn.Parameter:
"""Resolve checkpoint names without changing the model's module tree."""
module_name, _, parameter_name = name.rpartition(".")
module = model.get_submodule(module_name)
return module.get_parameter(parameter_name)
def reload_weights(
vllm_config: VllmConfig,
model: nn.Module,
weights_iterator: Iterable[tuple[str, torch.Tensor]] | None,
weights_path: str | None,
is_checkpoint_format: bool,
) -> None:
"""Reload a model's weights in place from a weights iterator or from disk.
Callers are responsible for invalidating any state derived from the old
weights (e.g. encoder and multimodal caches).
Args:
vllm_config: config of the model being reloaded. If `weights_path` is
given, `vllm_config.model_config` is updated to point at it.
model: model to load weights into
weights_iterator: weights to load into model
weights_path: path to load weights from if weights_iterator is not
provided. Use path of original model if neither is provided.
is_checkpoint_format: set to False if weights have already been
processed into kernel format (repacking, renaming, etc.)
"""
# TODO(@kylesayrs): generalize to all runners and loaders
# argument validation
if weights_iterator is None and not is_checkpoint_format:
logger.warning(
"Reloading from disk means that weights will be in checkpoint format. "
"Please use `is_checkpoint_format=True` "
"to avoid weight reloading errors"
)
model_config = vllm_config.model_config
load_config = vllm_config.load_config
weights_to_load = {name for name, _ in model.named_parameters()}
counter_before_reloading = time.perf_counter()
# load weights from disk if none are provided
if weights_iterator is None:
model_loader = get_model_loader(load_config)
if not hasattr(model_loader, "get_all_weights"):
raise NotImplementedError(
f"Model reloading with `{load_config.load_format}` format"
)
if weights_path is not None:
# The revision and any object-storage `model_weights` source
# belong to the model we are reloading away from, so they must
# not be carried over to the new path.
model_config.model = weights_path
model_config.model_weights = ""
model_config.revision = None
weights_iterator = model_loader.get_all_weights(model_config, model)
weights_iterator = cast(Iterable[tuple[str, torch.Tensor]], weights_iterator)
# begin loading weights
logger.info_once("Reloading weights inplace...")
if is_checkpoint_format:
# load weights from checkpoint/ original model format
initialize_layerwise_reload(model)
loaded_weights = model.load_weights(weights_iterator)
finalize_layerwise_reload(model, model_config)
else:
# load weights from kernel format
logger.warning_once(
"Reloading with `is_checkpoint_format=True` requires that "
"weights be in kernel format and already sharded",
)
loaded_weights = set()
for name, loaded_weight in weights_iterator:
param = get_parameter_for_reload(model, name) # TODO: buffers?
param.copy_(loaded_weight)
loaded_weights.add(name)
# logging and validation
counter_after_reloading = time.perf_counter()
diff_seconds = counter_after_reloading - counter_before_reloading
logger.info_once(
"Reloading and processing weights took %.2f seconds",
diff_seconds,
)
if model_config.quantization is None and loaded_weights is not None:
weights_not_loaded = weights_to_load - loaded_weights
if weights_not_loaded:
logger.warning(
"Following weights were not loaded from checkpoint: %s",
weights_not_loaded,
)