[Model Loader] Make dummy weight init dtype-aware (#61134)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
c6566bb9f6
commit
f47792af7f
@@ -6,13 +6,16 @@ from unittest.mock import Mock
|
||||
|
||||
import huggingface_hub.constants
|
||||
import pytest
|
||||
import torch
|
||||
from huggingface_hub.utils import LocalEntryNotFoundError
|
||||
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
download_weights_from_hf,
|
||||
drop_checkpoint_cache,
|
||||
initialize_single_dummy_weight,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def test_download_weights_from_hf():
|
||||
@@ -307,5 +310,47 @@ def test_drop_checkpoint_cache_releases_local_weight_files(
|
||||
fadvise.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float8_e4m3fn, torch.float8_e5m2])
|
||||
def test_dummy_fp8_weight_is_not_degenerate(dtype):
|
||||
"""+-1e-3 underflows in FP8, so 8-bit floats default to +-1."""
|
||||
param = torch.empty(4096, dtype=dtype)
|
||||
initialize_single_dummy_weight(param)
|
||||
values = param.float()
|
||||
assert values.isfinite().all()
|
||||
assert values.abs().max() <= 1.0
|
||||
assert (values != 0).float().mean() > 0.99
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("dtype", "name"),
|
||||
[(torch.float8_e8m0fnu, "weight_scale_inv"), (torch.uint8, "w13_weight_scale")],
|
||||
)
|
||||
def test_dummy_e8m0_scale_is_fixed_exponent(dtype, name):
|
||||
"""E8M0 scales, including uint8-stored ones, get a finite 2**-7."""
|
||||
param = torch.empty(64, dtype=dtype)
|
||||
initialize_single_dummy_weight(param, name=name)
|
||||
assert (param.view(torch.uint8) == 120).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.uint8, torch.int8, torch.float4_e2m1fn_x2])
|
||||
def test_dummy_packed_weight_gets_random_bytes(dtype):
|
||||
"""Packed weights are filled deterministically instead of left uninitialized."""
|
||||
# Some backends can't fill packed FP4 directly, so view zeroed bytes.
|
||||
params = [torch.zeros(4096, dtype=torch.uint8).view(dtype) for _ in range(2)]
|
||||
for param in params:
|
||||
initialize_single_dummy_weight(param, name="w13_weight")
|
||||
raw = params[0].view(torch.uint8)
|
||||
assert raw.unique().numel() > 250
|
||||
assert torch.equal(raw, params[1].view(torch.uint8))
|
||||
|
||||
|
||||
@pytest.mark.skipif(current_platform.is_rocm(), reason="ROCm zeroes integer params")
|
||||
def test_dummy_wide_integer_param_is_untouched():
|
||||
"""Wider integer params may be indices, so random values are unsafe."""
|
||||
param = torch.arange(16, dtype=torch.int32)
|
||||
initialize_single_dummy_weight(param, name="g_idx")
|
||||
assert torch.equal(param, torch.arange(16, dtype=torch.int32))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_download_weights_from_hf()
|
||||
|
||||
@@ -45,7 +45,7 @@ class DummyModelLoader(BaseModelLoader):
|
||||
# Exclude nonpersistent buffers, matching state_dict().
|
||||
for name, tensor in get_layer_tensors(layer).items():
|
||||
if name not in layer._non_persistent_buffers_set:
|
||||
initialize_single_dummy_weight(tensor)
|
||||
initialize_single_dummy_weight(tensor, name=name)
|
||||
|
||||
def _process_online_quant_layer(
|
||||
self,
|
||||
@@ -55,8 +55,8 @@ class DummyModelLoader(BaseModelLoader):
|
||||
"""Materialize, apply dummy weights, and run quantization processing."""
|
||||
materialize_layer(layer, info)
|
||||
|
||||
for tensor in get_layer_tensors(layer).values():
|
||||
initialize_single_dummy_weight(tensor)
|
||||
for name, tensor in get_layer_tensors(layer).items():
|
||||
initialize_single_dummy_weight(tensor, name=name)
|
||||
|
||||
for param in get_layer_tensors(layer).values():
|
||||
param.weight_loader = _get_original_loader(param)
|
||||
|
||||
@@ -1521,13 +1521,30 @@ def composed_weight_loader(
|
||||
return composed_loader
|
||||
|
||||
|
||||
# E8M0 byte 120 encodes 2**-7: with packed FP4 (|x| <= 6) or the FP8 dummy
|
||||
# range below, the dequantized weights stay small and finite.
|
||||
_DUMMY_E8M0_SCALE_BYTE = 120
|
||||
# Default dummy range of 8-bit floats. +-1e-3 is below FP8 E4M3's smallest
|
||||
# normal (2**-6), so it would mostly round to zero or subnormals.
|
||||
_DUMMY_FP8_RANGE = 1.0
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def initialize_single_dummy_weight(
|
||||
param: torch.Tensor,
|
||||
low: float = -1e-3,
|
||||
high: float = 1e-3,
|
||||
low: float | None = None,
|
||||
high: float | None = None,
|
||||
seed: int = 1234,
|
||||
name: str = "",
|
||||
) -> None:
|
||||
"""Fill `param` with deterministic dummy values suited to its dtype.
|
||||
|
||||
8-bit integer and packed FP4 params get random bytes (every FP4 code is
|
||||
finite), except uint8 params named like a scale, which hold E8M0 scales
|
||||
and get a fixed exponent. Other integer params (e.g. indices) are left
|
||||
untouched. Float params are sampled from `[low, high]`, which defaults to
|
||||
+-1e-3, or +-1 for 8-bit floats.
|
||||
"""
|
||||
if param.device.type == "meta":
|
||||
return # deferred to finalize_layerwise_processing (e.g. online quant)
|
||||
|
||||
@@ -1535,6 +1552,25 @@ def initialize_single_dummy_weight(
|
||||
param.fill_(dummy_weight_value)
|
||||
return
|
||||
|
||||
is_e8m0 = param.dtype == torch.float8_e8m0fnu or (
|
||||
param.dtype == torch.uint8 and "scale" in name
|
||||
)
|
||||
if is_e8m0:
|
||||
param.view(torch.uint8).fill_(_DUMMY_E8M0_SCALE_BYTE)
|
||||
return
|
||||
|
||||
if (
|
||||
param.dtype in (torch.uint8, torch.int8, torch.float4_e2m1fn_x2)
|
||||
and not current_platform.is_tpu()
|
||||
):
|
||||
generator = torch.Generator(device=param.device)
|
||||
generator.manual_seed(seed)
|
||||
if param.dtype == torch.int8:
|
||||
param.random_(-128, 128, generator=generator)
|
||||
else:
|
||||
param.view(torch.uint8).random_(0, 256, generator=generator)
|
||||
return
|
||||
|
||||
if not torch.is_floating_point(param):
|
||||
if current_platform.is_rocm():
|
||||
# On ROCm, integer params (e.g. GPTQ qweight/qzeros) are left
|
||||
@@ -1543,6 +1579,13 @@ def initialize_single_dummy_weight(
|
||||
param.zero_()
|
||||
return
|
||||
|
||||
if low is None or high is None:
|
||||
dummy_range = 1e-3
|
||||
if torch.finfo(param.dtype).bits < 16:
|
||||
dummy_range = _DUMMY_FP8_RANGE
|
||||
low = -dummy_range if low is None else low
|
||||
high = dummy_range if high is None else high
|
||||
|
||||
if current_platform.is_tpu():
|
||||
generator = torch.Generator(device="cpu")
|
||||
generator.manual_seed(seed)
|
||||
|
||||
Reference in New Issue
Block a user