[Bugfix][ModelLoader] Make enable_weights_track check quantized layers (#57108)

Signed-off-by: siliangchen-amd <SiLiang.Chen@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
siliangchen-amd
2026-10-10 10:44:06 +08:00
committed by GitHub
co-authored by Cursor Shanshan Shen
parent 1dfe9fbdae
commit 02aa10539f
3 changed files with 119 additions and 4 deletions
@@ -16,6 +16,7 @@ from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
from vllm.config import ModelConfig
from vllm.config.load import LoadConfig
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
from vllm.model_executor.layers.quantization.torchao import torchao_version_at_least
from vllm.model_executor.model_loader.base_loader import BaseModelLoader
from vllm.model_executor.model_loader.ep_weight_filter import (
@@ -47,6 +48,9 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
# fbgemm_fp8 builds input_scale_ub from its config; checkpoints do not carry it.
_CONFIG_BUILT_PARAMS = ("input_scale_ub",)
class DefaultModelLoader(BaseModelLoader):
"""Model loader that can load different file types from disk."""
@@ -506,10 +510,15 @@ class DefaultModelLoader(BaseModelLoader):
else default_enable_weights_track
)
if enable_weights_track:
self.track_weights_loading(model, loaded_weights)
self.track_weights_loading(
model, loaded_weights, quantized=model_config.quantization is not None
)
def track_weights_loading(
self, model: nn.Module, loaded_weights: set[str] | None
self,
model: nn.Module,
loaded_weights: set[str] | None,
quantized: bool = False,
) -> None:
weights_to_load = {name for name, _ in model.named_parameters()}
if loaded_weights is not None:
@@ -523,7 +532,28 @@ class DefaultModelLoader(BaseModelLoader):
# ignore kv_cache scale and online quant scale,
# which can be missing in checkpoints
if has_online_quant or has_postprocess_quant:
for param_name, _ in module.named_parameters():
for param_name, param in module.named_parameters():
# On a quantized model, every parameter has to come from
# the checkpoint except those a checkpoint cannot carry:
# - online quantization (uses_meta_device) quantizes the
# weights while loading, so its parameters do not map
# to checkpoint tensors;
# - KV-cache quantization parameters (BaseKVCacheMethod)
# fall back to defaults when a checkpoint has none;
# - empty placeholders hold no data, e.g. the qzeros
# moe_wna16 registers for symmetric GPTQ;
# - _CONFIG_BUILT_PARAMS come from the quantization config.
# Unquantized models, where this check runs by default,
# keep the module-wide exemption.
if (
quantized
and not has_online_quant
and not isinstance(quant_method, BaseKVCacheMethod)
and param.numel() > 0
and param_name.rsplit(".", 1)[-1]
not in _CONFIG_BUILT_PARAMS
):
continue
full_name = f"{name}.{param_name}" if name else param_name
loaded_weights.add(full_name)
weights_not_loaded = weights_to_load - loaded_weights