[Bugfix] Avoid repeated dummy initialization and random CPU Engram fills (#56682)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
Woosuk Kwon
2026-09-13 02:23:20 -07:00
committed by GitHub
parent a987777755
commit e7a3963339
3 changed files with 14 additions and 2 deletions
@@ -14,7 +14,6 @@ from vllm.model_executor.model_loader.reload.meta import materialize_layer
from vllm.model_executor.model_loader.reload.types import LayerReloadingInfo
from vllm.model_executor.model_loader.reload.utils import get_layer_tensors
from vllm.model_executor.model_loader.weight_utils import (
initialize_dummy_weights,
initialize_single_dummy_weight,
)
@@ -41,7 +40,12 @@ class DummyModelLoader(BaseModelLoader):
else:
# NOTE(woosuk): For accurate performance evaluation, we assign
# random values to the weights.
initialize_dummy_weights(layer, model_config)
# Initialize only local tensors: state_dict() would recursively
# initialize descendants that model.modules() also visits.
# Exclude nonpersistent buffers, matching state_dict().
for name, tensor in get_layer_tensors(layer).items():
if name not in layer._non_persistent_buffers_set:
initialize_single_dummy_weight(tensor)
def _process_online_quant_layer(
self,
@@ -1341,6 +1341,10 @@ def initialize_single_dummy_weight(
if param.device.type == "meta":
return # deferred to finalize_layerwise_processing (e.g. online quant)
if (dummy_weight_value := getattr(param, "dummy_weight_value", None)) is not None:
param.fill_(dummy_weight_value)
return
if not torch.is_floating_point(param):
if current_platform.is_rocm():
# On ROCm, integer params (e.g. GPTQ qweight/qzeros) are left
@@ -687,6 +687,10 @@ class ParallelEngramEmbedding(nn.Module):
},
)
if cpu_offload:
# Constant dummy values avoid randomizing huge CPU lookup tables.
set_weight_attrs(self.weight, {"dummy_weight_value": 1.0})
# The ue8m0 encoding of scale 1.0 is exponent byte 127.
set_weight_attrs(self.weight_scale_inv, {"dummy_weight_value": 127})
logger.info(
"Engram table offloaded to pinned host memory: %d rows x %d, "
"%.2f GiB per rank",