Fast Start (#54921)
Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: liusy58 <liusy58@smail.nju.edu.cn> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -26,6 +26,7 @@ from vllm.model_executor.model_loader.reload import (
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_tying import maybe_retie_word_embeddings
|
||||
from vllm.model_executor.models.interfaces import SupportsQuant
|
||||
from vllm.model_executor.utils import is_weights_pre_processed
|
||||
from vllm.tracing import instrument
|
||||
from vllm.utils.mem_utils import release_device_memory_under_pressure
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
@@ -97,13 +98,27 @@ def initialize_model(
|
||||
def process_weights_after_loading(
|
||||
model: nn.Module, model_config: ModelConfig, target_device: torch.device
|
||||
) -> None:
|
||||
"""Post-process loaded weights into runtime format.
|
||||
|
||||
Under ``weights_already_processed`` (weight cache IPC loader), quant
|
||||
methods skip tensor transforms and must declare
|
||||
``supports_pre_processed_weights``, otherwise this raises ``RuntimeError``.
|
||||
"""
|
||||
# Reclaim memory when an explicit lm_head has been
|
||||
# loaded, but it is identical to the input embeddings.
|
||||
maybe_retie_word_embeddings(model, model_config)
|
||||
|
||||
for _, module in model.named_modules():
|
||||
for name, module in model.named_modules():
|
||||
quant_method = getattr(module, "quant_method", None)
|
||||
if isinstance(quant_method, QuantizeMethodBase):
|
||||
if (
|
||||
is_weights_pre_processed()
|
||||
and not quant_method.supports_pre_processed_weights
|
||||
):
|
||||
raise RuntimeError(
|
||||
f"layer {name or '<root>'}: {type(quant_method).__name__} "
|
||||
"does not support pre-processed weights"
|
||||
)
|
||||
# When quant methods need to process weights after loading
|
||||
# (for repacking, quantizing, etc), they expect parameters
|
||||
# to be on the global target device. This scope is for the
|
||||
|
||||
Reference in New Issue
Block a user