[Bugfix][ModelLoader] Make enable_weights_track check quantized layers (#57108)
Signed-off-by: siliangchen-amd <SiLiang.Chen@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
co-authored by
Cursor
Shanshan Shen
parent
1dfe9fbdae
commit
02aa10539f
@@ -301,7 +301,10 @@ class MoERunner(MoERunnerInterface):
|
||||
def load_weights(
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[str]:
|
||||
return self.routed_experts.load_weights(weights)
|
||||
# routed_experts reports names relative to itself, but callers such as
|
||||
# AutoWeightsLoader qualify what this returns with the runner's prefix.
|
||||
for name in self.routed_experts.load_weights(weights):
|
||||
yield f"routed_experts.{name}"
|
||||
|
||||
def _select_forward(self) -> Callable:
|
||||
if current_platform.is_tpu():
|
||||
|
||||
@@ -16,6 +16,7 @@ from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.config.load import LoadConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
|
||||
from vllm.model_executor.layers.quantization.torchao import torchao_version_at_least
|
||||
from vllm.model_executor.model_loader.base_loader import BaseModelLoader
|
||||
from vllm.model_executor.model_loader.ep_weight_filter import (
|
||||
@@ -47,6 +48,9 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# fbgemm_fp8 builds input_scale_ub from its config; checkpoints do not carry it.
|
||||
_CONFIG_BUILT_PARAMS = ("input_scale_ub",)
|
||||
|
||||
|
||||
class DefaultModelLoader(BaseModelLoader):
|
||||
"""Model loader that can load different file types from disk."""
|
||||
@@ -506,10 +510,15 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
else default_enable_weights_track
|
||||
)
|
||||
if enable_weights_track:
|
||||
self.track_weights_loading(model, loaded_weights)
|
||||
self.track_weights_loading(
|
||||
model, loaded_weights, quantized=model_config.quantization is not None
|
||||
)
|
||||
|
||||
def track_weights_loading(
|
||||
self, model: nn.Module, loaded_weights: set[str] | None
|
||||
self,
|
||||
model: nn.Module,
|
||||
loaded_weights: set[str] | None,
|
||||
quantized: bool = False,
|
||||
) -> None:
|
||||
weights_to_load = {name for name, _ in model.named_parameters()}
|
||||
if loaded_weights is not None:
|
||||
@@ -523,7 +532,28 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
# ignore kv_cache scale and online quant scale,
|
||||
# which can be missing in checkpoints
|
||||
if has_online_quant or has_postprocess_quant:
|
||||
for param_name, _ in module.named_parameters():
|
||||
for param_name, param in module.named_parameters():
|
||||
# On a quantized model, every parameter has to come from
|
||||
# the checkpoint except those a checkpoint cannot carry:
|
||||
# - online quantization (uses_meta_device) quantizes the
|
||||
# weights while loading, so its parameters do not map
|
||||
# to checkpoint tensors;
|
||||
# - KV-cache quantization parameters (BaseKVCacheMethod)
|
||||
# fall back to defaults when a checkpoint has none;
|
||||
# - empty placeholders hold no data, e.g. the qzeros
|
||||
# moe_wna16 registers for symmetric GPTQ;
|
||||
# - _CONFIG_BUILT_PARAMS come from the quantization config.
|
||||
# Unquantized models, where this check runs by default,
|
||||
# keep the module-wide exemption.
|
||||
if (
|
||||
quantized
|
||||
and not has_online_quant
|
||||
and not isinstance(quant_method, BaseKVCacheMethod)
|
||||
and param.numel() > 0
|
||||
and param_name.rsplit(".", 1)[-1]
|
||||
not in _CONFIG_BUILT_PARAMS
|
||||
):
|
||||
continue
|
||||
full_name = f"{name}.{param_name}" if name else param_name
|
||||
loaded_weights.add(full_name)
|
||||
weights_not_loaded = weights_to_load - loaded_weights
|
||||
|
||||
Reference in New Issue
Block a user