From f47792af7fba76eb8e8f6a564b35cfad316d39f2 Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Mon, 12 Oct 2026 18:33:26 -0700 Subject: [PATCH] [Model Loader] Make dummy weight init dtype-aware (#61134) Signed-off-by: Woosuk Kwon Co-authored-by: Claude Opus 5.5 --- tests/model_executor/test_weight_utils.py | 45 ++++++++++++++++++ .../model_loader/dummy_loader.py | 6 +-- .../model_loader/weight_utils.py | 47 ++++++++++++++++++- 3 files changed, 93 insertions(+), 5 deletions(-) diff --git a/tests/model_executor/test_weight_utils.py b/tests/model_executor/test_weight_utils.py index 512f518a05..12a8e3acac 100644 --- a/tests/model_executor/test_weight_utils.py +++ b/tests/model_executor/test_weight_utils.py @@ -6,13 +6,16 @@ from unittest.mock import Mock import huggingface_hub.constants import pytest +import torch from huggingface_hub.utils import LocalEntryNotFoundError from vllm.model_executor.model_loader.weight_utils import ( download_weights_from_hf, drop_checkpoint_cache, + initialize_single_dummy_weight, maybe_remap_kv_scale_name, ) +from vllm.platforms import current_platform def test_download_weights_from_hf(): @@ -307,5 +310,47 @@ def test_drop_checkpoint_cache_releases_local_weight_files( fadvise.assert_called_once() +@pytest.mark.parametrize("dtype", [torch.float8_e4m3fn, torch.float8_e5m2]) +def test_dummy_fp8_weight_is_not_degenerate(dtype): + """+-1e-3 underflows in FP8, so 8-bit floats default to +-1.""" + param = torch.empty(4096, dtype=dtype) + initialize_single_dummy_weight(param) + values = param.float() + assert values.isfinite().all() + assert values.abs().max() <= 1.0 + assert (values != 0).float().mean() > 0.99 + + +@pytest.mark.parametrize( + ("dtype", "name"), + [(torch.float8_e8m0fnu, "weight_scale_inv"), (torch.uint8, "w13_weight_scale")], +) +def test_dummy_e8m0_scale_is_fixed_exponent(dtype, name): + """E8M0 scales, including uint8-stored ones, get a finite 2**-7.""" + param = torch.empty(64, dtype=dtype) + initialize_single_dummy_weight(param, name=name) + assert (param.view(torch.uint8) == 120).all() + + +@pytest.mark.parametrize("dtype", [torch.uint8, torch.int8, torch.float4_e2m1fn_x2]) +def test_dummy_packed_weight_gets_random_bytes(dtype): + """Packed weights are filled deterministically instead of left uninitialized.""" + # Some backends can't fill packed FP4 directly, so view zeroed bytes. + params = [torch.zeros(4096, dtype=torch.uint8).view(dtype) for _ in range(2)] + for param in params: + initialize_single_dummy_weight(param, name="w13_weight") + raw = params[0].view(torch.uint8) + assert raw.unique().numel() > 250 + assert torch.equal(raw, params[1].view(torch.uint8)) + + +@pytest.mark.skipif(current_platform.is_rocm(), reason="ROCm zeroes integer params") +def test_dummy_wide_integer_param_is_untouched(): + """Wider integer params may be indices, so random values are unsafe.""" + param = torch.arange(16, dtype=torch.int32) + initialize_single_dummy_weight(param, name="g_idx") + assert torch.equal(param, torch.arange(16, dtype=torch.int32)) + + if __name__ == "__main__": test_download_weights_from_hf() diff --git a/vllm/model_executor/model_loader/dummy_loader.py b/vllm/model_executor/model_loader/dummy_loader.py index 0f263fe117..ff732f6b22 100644 --- a/vllm/model_executor/model_loader/dummy_loader.py +++ b/vllm/model_executor/model_loader/dummy_loader.py @@ -45,7 +45,7 @@ class DummyModelLoader(BaseModelLoader): # Exclude nonpersistent buffers, matching state_dict(). for name, tensor in get_layer_tensors(layer).items(): if name not in layer._non_persistent_buffers_set: - initialize_single_dummy_weight(tensor) + initialize_single_dummy_weight(tensor, name=name) def _process_online_quant_layer( self, @@ -55,8 +55,8 @@ class DummyModelLoader(BaseModelLoader): """Materialize, apply dummy weights, and run quantization processing.""" materialize_layer(layer, info) - for tensor in get_layer_tensors(layer).values(): - initialize_single_dummy_weight(tensor) + for name, tensor in get_layer_tensors(layer).items(): + initialize_single_dummy_weight(tensor, name=name) for param in get_layer_tensors(layer).values(): param.weight_loader = _get_original_loader(param) diff --git a/vllm/model_executor/model_loader/weight_utils.py b/vllm/model_executor/model_loader/weight_utils.py index ac1da8d698..0a834cd90c 100644 --- a/vllm/model_executor/model_loader/weight_utils.py +++ b/vllm/model_executor/model_loader/weight_utils.py @@ -1521,13 +1521,30 @@ def composed_weight_loader( return composed_loader +# E8M0 byte 120 encodes 2**-7: with packed FP4 (|x| <= 6) or the FP8 dummy +# range below, the dequantized weights stay small and finite. +_DUMMY_E8M0_SCALE_BYTE = 120 +# Default dummy range of 8-bit floats. +-1e-3 is below FP8 E4M3's smallest +# normal (2**-6), so it would mostly round to zero or subnormals. +_DUMMY_FP8_RANGE = 1.0 + + @torch.no_grad() def initialize_single_dummy_weight( param: torch.Tensor, - low: float = -1e-3, - high: float = 1e-3, + low: float | None = None, + high: float | None = None, seed: int = 1234, + name: str = "", ) -> None: + """Fill `param` with deterministic dummy values suited to its dtype. + + 8-bit integer and packed FP4 params get random bytes (every FP4 code is + finite), except uint8 params named like a scale, which hold E8M0 scales + and get a fixed exponent. Other integer params (e.g. indices) are left + untouched. Float params are sampled from `[low, high]`, which defaults to + +-1e-3, or +-1 for 8-bit floats. + """ if param.device.type == "meta": return # deferred to finalize_layerwise_processing (e.g. online quant) @@ -1535,6 +1552,25 @@ def initialize_single_dummy_weight( param.fill_(dummy_weight_value) return + is_e8m0 = param.dtype == torch.float8_e8m0fnu or ( + param.dtype == torch.uint8 and "scale" in name + ) + if is_e8m0: + param.view(torch.uint8).fill_(_DUMMY_E8M0_SCALE_BYTE) + return + + if ( + param.dtype in (torch.uint8, torch.int8, torch.float4_e2m1fn_x2) + and not current_platform.is_tpu() + ): + generator = torch.Generator(device=param.device) + generator.manual_seed(seed) + if param.dtype == torch.int8: + param.random_(-128, 128, generator=generator) + else: + param.view(torch.uint8).random_(0, 256, generator=generator) + return + if not torch.is_floating_point(param): if current_platform.is_rocm(): # On ROCm, integer params (e.g. GPTQ qweight/qzeros) are left @@ -1543,6 +1579,13 @@ def initialize_single_dummy_weight( param.zero_() return + if low is None or high is None: + dummy_range = 1e-3 + if torch.finfo(param.dtype).bits < 16: + dummy_range = _DUMMY_FP8_RANGE + low = -dummy_range if low is None else low + high = dummy_range if high is None else high + if current_platform.is_tpu(): generator = torch.Generator(device="cpu") generator.manual_seed(seed)