[CI] Speed up quantization test group (#53291)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Andreas Karatzas
parent
e7cf4730d6
commit
d1c15e589d
@@ -2,13 +2,83 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
|
||||
from vllm.config import (
|
||||
CompilationConfig,
|
||||
ModelConfig,
|
||||
VllmConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import get_quantization_config
|
||||
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _limit_num_hidden_layers(
|
||||
model: torch.nn.Module, num_hidden_layers: int | None
|
||||
) -> None:
|
||||
if num_hidden_layers is None:
|
||||
return
|
||||
|
||||
original_load_weights = model.load_weights
|
||||
|
||||
def should_load_weight(name: str) -> bool:
|
||||
for prefix in ("model.layers.", "layers."):
|
||||
if name.startswith(prefix):
|
||||
layer_idx = int(name.removeprefix(prefix).split(".", 1)[0])
|
||||
return layer_idx < num_hidden_layers
|
||||
return True
|
||||
|
||||
def load_weights(weights):
|
||||
weights = (
|
||||
(name, weight) for name, weight in weights if should_load_weight(name)
|
||||
)
|
||||
return original_load_weights(weights)
|
||||
|
||||
model.load_weights = load_weights
|
||||
|
||||
|
||||
def load_model_without_vllm_runner(
|
||||
model_path: str,
|
||||
*,
|
||||
dtype: str | torch.dtype = "bfloat16",
|
||||
quantization: str | None = None,
|
||||
model_config_kwargs: dict[str, Any] | None = None,
|
||||
vllm_config_kwargs: dict[str, Any] | None = None,
|
||||
model_loader_cls: type = DefaultModelLoader,
|
||||
) -> tuple[torch.nn.Module, VllmConfig]:
|
||||
"""Instantiate a model, load weights, and process them for inference."""
|
||||
model_config = ModelConfig(
|
||||
model=model_path,
|
||||
dtype=dtype,
|
||||
quantization=quantization,
|
||||
**(model_config_kwargs or {}),
|
||||
)
|
||||
vllm_config_args = dict(vllm_config_kwargs or {})
|
||||
vllm_config_args.setdefault("compilation_config", CompilationConfig(mode=0))
|
||||
vllm_config = VllmConfig(model_config=model_config, **vllm_config_args)
|
||||
hf_overrides = (model_config_kwargs or {}).get("hf_overrides") or {}
|
||||
num_hidden_layers = hf_overrides.get("num_hidden_layers")
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
model_loader = model_loader_cls(vllm_config.load_config)
|
||||
if num_hidden_layers is not None:
|
||||
original_load_weights = model_loader.load_weights
|
||||
|
||||
def load_weights(model, model_config):
|
||||
_limit_num_hidden_layers(model, num_hidden_layers)
|
||||
original_load_weights(model, model_config)
|
||||
|
||||
model_loader.load_weights = load_weights
|
||||
model = model_loader.load_model(vllm_config, model_config)
|
||||
|
||||
return model, vllm_config
|
||||
|
||||
|
||||
def is_quant_method_supported(quant_method: str) -> bool:
|
||||
# Currently, quantization tests only run GPUs
|
||||
if current_platform.is_cpu():
|
||||
|
||||
Reference in New Issue
Block a user