[Model Loader] Make dummy weight init dtype-aware (#61134)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Woosuk Kwon
2026-10-12 18:33:26 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent c6566bb9f6
commit f47792af7f
3 changed files with 93 additions and 5 deletions
+45
View File
@@ -6,13 +6,16 @@ from unittest.mock import Mock
import huggingface_hub.constants
import pytest
import torch
from huggingface_hub.utils import LocalEntryNotFoundError
from vllm.model_executor.model_loader.weight_utils import (
download_weights_from_hf,
drop_checkpoint_cache,
initialize_single_dummy_weight,
maybe_remap_kv_scale_name,
)
from vllm.platforms import current_platform
def test_download_weights_from_hf():
@@ -307,5 +310,47 @@ def test_drop_checkpoint_cache_releases_local_weight_files(
fadvise.assert_called_once()
@pytest.mark.parametrize("dtype", [torch.float8_e4m3fn, torch.float8_e5m2])
def test_dummy_fp8_weight_is_not_degenerate(dtype):
"""+-1e-3 underflows in FP8, so 8-bit floats default to +-1."""
param = torch.empty(4096, dtype=dtype)
initialize_single_dummy_weight(param)
values = param.float()
assert values.isfinite().all()
assert values.abs().max() <= 1.0
assert (values != 0).float().mean() > 0.99
@pytest.mark.parametrize(
("dtype", "name"),
[(torch.float8_e8m0fnu, "weight_scale_inv"), (torch.uint8, "w13_weight_scale")],
)
def test_dummy_e8m0_scale_is_fixed_exponent(dtype, name):
"""E8M0 scales, including uint8-stored ones, get a finite 2**-7."""
param = torch.empty(64, dtype=dtype)
initialize_single_dummy_weight(param, name=name)
assert (param.view(torch.uint8) == 120).all()
@pytest.mark.parametrize("dtype", [torch.uint8, torch.int8, torch.float4_e2m1fn_x2])
def test_dummy_packed_weight_gets_random_bytes(dtype):
"""Packed weights are filled deterministically instead of left uninitialized."""
# Some backends can't fill packed FP4 directly, so view zeroed bytes.
params = [torch.zeros(4096, dtype=torch.uint8).view(dtype) for _ in range(2)]
for param in params:
initialize_single_dummy_weight(param, name="w13_weight")
raw = params[0].view(torch.uint8)
assert raw.unique().numel() > 250
assert torch.equal(raw, params[1].view(torch.uint8))
@pytest.mark.skipif(current_platform.is_rocm(), reason="ROCm zeroes integer params")
def test_dummy_wide_integer_param_is_untouched():
"""Wider integer params may be indices, so random values are unsafe."""
param = torch.arange(16, dtype=torch.int32)
initialize_single_dummy_weight(param, name="g_idx")
assert torch.equal(param, torch.arange(16, dtype=torch.int32))
if __name__ == "__main__":
test_download_weights_from_hf()
@@ -45,7 +45,7 @@ class DummyModelLoader(BaseModelLoader):
# Exclude nonpersistent buffers, matching state_dict().
for name, tensor in get_layer_tensors(layer).items():
if name not in layer._non_persistent_buffers_set:
initialize_single_dummy_weight(tensor)
initialize_single_dummy_weight(tensor, name=name)
def _process_online_quant_layer(
self,
@@ -55,8 +55,8 @@ class DummyModelLoader(BaseModelLoader):
"""Materialize, apply dummy weights, and run quantization processing."""
materialize_layer(layer, info)
for tensor in get_layer_tensors(layer).values():
initialize_single_dummy_weight(tensor)
for name, tensor in get_layer_tensors(layer).items():
initialize_single_dummy_weight(tensor, name=name)
for param in get_layer_tensors(layer).values():
param.weight_loader = _get_original_loader(param)
@@ -1521,13 +1521,30 @@ def composed_weight_loader(
return composed_loader
# E8M0 byte 120 encodes 2**-7: with packed FP4 (|x| <= 6) or the FP8 dummy
# range below, the dequantized weights stay small and finite.
_DUMMY_E8M0_SCALE_BYTE = 120
# Default dummy range of 8-bit floats. +-1e-3 is below FP8 E4M3's smallest
# normal (2**-6), so it would mostly round to zero or subnormals.
_DUMMY_FP8_RANGE = 1.0
@torch.no_grad()
def initialize_single_dummy_weight(
param: torch.Tensor,
low: float = -1e-3,
high: float = 1e-3,
low: float | None = None,
high: float | None = None,
seed: int = 1234,
name: str = "",
) -> None:
"""Fill `param` with deterministic dummy values suited to its dtype.
8-bit integer and packed FP4 params get random bytes (every FP4 code is
finite), except uint8 params named like a scale, which hold E8M0 scales
and get a fixed exponent. Other integer params (e.g. indices) are left
untouched. Float params are sampled from `[low, high]`, which defaults to
+-1e-3, or +-1 for 8-bit floats.
"""
if param.device.type == "meta":
return # deferred to finalize_layerwise_processing (e.g. online quant)
@@ -1535,6 +1552,25 @@ def initialize_single_dummy_weight(
param.fill_(dummy_weight_value)
return
is_e8m0 = param.dtype == torch.float8_e8m0fnu or (
param.dtype == torch.uint8 and "scale" in name
)
if is_e8m0:
param.view(torch.uint8).fill_(_DUMMY_E8M0_SCALE_BYTE)
return
if (
param.dtype in (torch.uint8, torch.int8, torch.float4_e2m1fn_x2)
and not current_platform.is_tpu()
):
generator = torch.Generator(device=param.device)
generator.manual_seed(seed)
if param.dtype == torch.int8:
param.random_(-128, 128, generator=generator)
else:
param.view(torch.uint8).random_(0, 256, generator=generator)
return
if not torch.is_floating_point(param):
if current_platform.is_rocm():
# On ROCm, integer params (e.g. GPTQ qweight/qzeros) are left
@@ -1543,6 +1579,13 @@ def initialize_single_dummy_weight(
param.zero_()
return
if low is None or high is None:
dummy_range = 1e-3
if torch.finfo(param.dtype).bits < 16:
dummy_range = _DUMMY_FP8_RANGE
low = -dummy_range if low is None else low
high = dummy_range if high is None else high
if current_platform.is_tpu():
generator = torch.Generator(device="cpu")
generator.manual_seed(seed)