[CI] Speed up quantization test group (#53291)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
fxmarty-amd
2026-09-01 13:15:24 -05:00
committed by GitHub
co-authored by Andreas Karatzas
parent e7cf4730d6
commit d1c15e589d
20 changed files with 824 additions and 725 deletions
+70
View File
@@ -2,13 +2,83 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import logging
from typing import Any
import regex as re
import torch
from vllm.config import (
CompilationConfig,
ModelConfig,
VllmConfig,
set_current_vllm_config,
)
from vllm.model_executor.layers.quantization import get_quantization_config
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
from vllm.platforms import current_platform
def _limit_num_hidden_layers(
model: torch.nn.Module, num_hidden_layers: int | None
) -> None:
if num_hidden_layers is None:
return
original_load_weights = model.load_weights
def should_load_weight(name: str) -> bool:
for prefix in ("model.layers.", "layers."):
if name.startswith(prefix):
layer_idx = int(name.removeprefix(prefix).split(".", 1)[0])
return layer_idx < num_hidden_layers
return True
def load_weights(weights):
weights = (
(name, weight) for name, weight in weights if should_load_weight(name)
)
return original_load_weights(weights)
model.load_weights = load_weights
def load_model_without_vllm_runner(
model_path: str,
*,
dtype: str | torch.dtype = "bfloat16",
quantization: str | None = None,
model_config_kwargs: dict[str, Any] | None = None,
vllm_config_kwargs: dict[str, Any] | None = None,
model_loader_cls: type = DefaultModelLoader,
) -> tuple[torch.nn.Module, VllmConfig]:
"""Instantiate a model, load weights, and process them for inference."""
model_config = ModelConfig(
model=model_path,
dtype=dtype,
quantization=quantization,
**(model_config_kwargs or {}),
)
vllm_config_args = dict(vllm_config_kwargs or {})
vllm_config_args.setdefault("compilation_config", CompilationConfig(mode=0))
vllm_config = VllmConfig(model_config=model_config, **vllm_config_args)
hf_overrides = (model_config_kwargs or {}).get("hf_overrides") or {}
num_hidden_layers = hf_overrides.get("num_hidden_layers")
with set_current_vllm_config(vllm_config):
model_loader = model_loader_cls(vllm_config.load_config)
if num_hidden_layers is not None:
original_load_weights = model_loader.load_weights
def load_weights(model, model_config):
_limit_num_hidden_layers(model, num_hidden_layers)
original_load_weights(model, model_config)
model_loader.load_weights = load_weights
model = model_loader.load_model(vllm_config, model_config)
return model, vllm_config
def is_quant_method_supported(quant_method: str) -> bool:
# Currently, quantization tests only run GPUs
if current_platform.is_cpu():