[Bug fix][Quantization] Fix dummy weight loading (#38478)

Signed-off-by: Josephasafg <ajgard7@gmail.com>
This commit is contained in:
Asaf Gardin
2026-03-30 16:38:02 -04:00
committed by GitHub
parent d9c7db18da
commit 1fc69f59bb
3 changed files with 19 additions and 13 deletions
@@ -11,7 +11,10 @@ from vllm.config import ModelConfig
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention, MLAAttention
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
initialize_single_dummy_weight,
)
from .meta import (
capture_layer_to_meta,
@@ -223,7 +226,8 @@ def finalize_layerwise_processing(model: torch.nn.Module, model_config: ModelCon
elif info.load_numel <= 0:
# first load but received no weights. This happens on dummy load
if info.kernel_tensors is None:
materialize_layer(layer, info)
_layerwise_process(layer, info)
continue
# reloading: place kernel tensors back as a fallback
else:
@@ -258,6 +262,12 @@ def _layerwise_process(layer: torch.nn.Module, info: LayerReloadingInfo):
# Materialize layer tensors onto device
materialize_layer(layer, info)
# If no weights were loaded (e.g. dummy loading), initialize with
# small random values to avoid NaN from zero/garbage data
if len(info.loaded_weights) <= 0:
for tensor in get_layer_tensors(layer).values():
initialize_single_dummy_weight(tensor)
# Reset online quantization flag so process_weights_after_loading
# will run again during reload
if hasattr(layer, "_already_called_process_weights_after_loading"):