Fast Start (#54921)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: liusy58 <liusy58@smail.nju.edu.cn>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
siyu
2026-09-04 06:32:06 -07:00
committed by GitHub
co-authored by liusy58 Isotr0py
parent 07ee97e023
commit a69e75b9b6
16 changed files with 1529 additions and 13 deletions
+16 -1
View File
@@ -26,6 +26,7 @@ from vllm.model_executor.model_loader.reload import (
)
from vllm.model_executor.model_loader.weight_tying import maybe_retie_word_embeddings
from vllm.model_executor.models.interfaces import SupportsQuant
from vllm.model_executor.utils import is_weights_pre_processed
from vllm.tracing import instrument
from vllm.utils.mem_utils import release_device_memory_under_pressure
from vllm.utils.platform_utils import is_pin_memory_available
@@ -97,13 +98,27 @@ def initialize_model(
def process_weights_after_loading(
model: nn.Module, model_config: ModelConfig, target_device: torch.device
) -> None:
"""Post-process loaded weights into runtime format.
Under ``weights_already_processed`` (weight cache IPC loader), quant
methods skip tensor transforms and must declare
``supports_pre_processed_weights``, otherwise this raises ``RuntimeError``.
"""
# Reclaim memory when an explicit lm_head has been
# loaded, but it is identical to the input embeddings.
maybe_retie_word_embeddings(model, model_config)
for _, module in model.named_modules():
for name, module in model.named_modules():
quant_method = getattr(module, "quant_method", None)
if isinstance(quant_method, QuantizeMethodBase):
if (
is_weights_pre_processed()
and not quant_method.supports_pre_processed_weights
):
raise RuntimeError(
f"layer {name or '<root>'}: {type(quant_method).__name__} "
"does not support pre-processed weights"
)
# When quant methods need to process weights after loading
# (for repacking, quantizing, etc), they expect parameters
# to be on the global target device. This scope is for the