[Model Loader] Make dummy weight init dtype-aware (#61134)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Woosuk Kwon
2026-10-12 18:33:26 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent c6566bb9f6
commit f47792af7f
3 changed files with 93 additions and 5 deletions
@@ -45,7 +45,7 @@ class DummyModelLoader(BaseModelLoader):
# Exclude nonpersistent buffers, matching state_dict().
for name, tensor in get_layer_tensors(layer).items():
if name not in layer._non_persistent_buffers_set:
initialize_single_dummy_weight(tensor)
initialize_single_dummy_weight(tensor, name=name)
def _process_online_quant_layer(
self,
@@ -55,8 +55,8 @@ class DummyModelLoader(BaseModelLoader):
"""Materialize, apply dummy weights, and run quantization processing."""
materialize_layer(layer, info)
for tensor in get_layer_tensors(layer).values():
initialize_single_dummy_weight(tensor)
for name, tensor in get_layer_tensors(layer).items():
initialize_single_dummy_weight(tensor, name=name)
for param in get_layer_tensors(layer).values():
param.weight_loader = _get_original_loader(param)
@@ -1521,13 +1521,30 @@ def composed_weight_loader(
return composed_loader
# E8M0 byte 120 encodes 2**-7: with packed FP4 (|x| <= 6) or the FP8 dummy
# range below, the dequantized weights stay small and finite.
_DUMMY_E8M0_SCALE_BYTE = 120
# Default dummy range of 8-bit floats. +-1e-3 is below FP8 E4M3's smallest
# normal (2**-6), so it would mostly round to zero or subnormals.
_DUMMY_FP8_RANGE = 1.0
@torch.no_grad()
def initialize_single_dummy_weight(
param: torch.Tensor,
low: float = -1e-3,
high: float = 1e-3,
low: float | None = None,
high: float | None = None,
seed: int = 1234,
name: str = "",
) -> None:
"""Fill `param` with deterministic dummy values suited to its dtype.
8-bit integer and packed FP4 params get random bytes (every FP4 code is
finite), except uint8 params named like a scale, which hold E8M0 scales
and get a fixed exponent. Other integer params (e.g. indices) are left
untouched. Float params are sampled from `[low, high]`, which defaults to
+-1e-3, or +-1 for 8-bit floats.
"""
if param.device.type == "meta":
return # deferred to finalize_layerwise_processing (e.g. online quant)
@@ -1535,6 +1552,25 @@ def initialize_single_dummy_weight(
param.fill_(dummy_weight_value)
return
is_e8m0 = param.dtype == torch.float8_e8m0fnu or (
param.dtype == torch.uint8 and "scale" in name
)
if is_e8m0:
param.view(torch.uint8).fill_(_DUMMY_E8M0_SCALE_BYTE)
return
if (
param.dtype in (torch.uint8, torch.int8, torch.float4_e2m1fn_x2)
and not current_platform.is_tpu()
):
generator = torch.Generator(device=param.device)
generator.manual_seed(seed)
if param.dtype == torch.int8:
param.random_(-128, 128, generator=generator)
else:
param.view(torch.uint8).random_(0, 256, generator=generator)
return
if not torch.is_floating_point(param):
if current_platform.is_rocm():
# On ROCm, integer params (e.g. GPTQ qweight/qzeros) are left
@@ -1543,6 +1579,13 @@ def initialize_single_dummy_weight(
param.zero_()
return
if low is None or high is None:
dummy_range = 1e-3
if torch.finfo(param.dtype).bits < 16:
dummy_range = _DUMMY_FP8_RANGE
low = -dummy_range if low is None else low
high = dummy_range if high is None else high
if current_platform.is_tpu():
generator = torch.Generator(device="cpu")
generator.manual_seed(seed)