[Bugfix] Avoid repeated dummy initialization and random CPU Engram fills (#56682)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
@@ -14,7 +14,6 @@ from vllm.model_executor.model_loader.reload.meta import materialize_layer
|
||||
from vllm.model_executor.model_loader.reload.types import LayerReloadingInfo
|
||||
from vllm.model_executor.model_loader.reload.utils import get_layer_tensors
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
initialize_dummy_weights,
|
||||
initialize_single_dummy_weight,
|
||||
)
|
||||
|
||||
@@ -41,7 +40,12 @@ class DummyModelLoader(BaseModelLoader):
|
||||
else:
|
||||
# NOTE(woosuk): For accurate performance evaluation, we assign
|
||||
# random values to the weights.
|
||||
initialize_dummy_weights(layer, model_config)
|
||||
# Initialize only local tensors: state_dict() would recursively
|
||||
# initialize descendants that model.modules() also visits.
|
||||
# Exclude nonpersistent buffers, matching state_dict().
|
||||
for name, tensor in get_layer_tensors(layer).items():
|
||||
if name not in layer._non_persistent_buffers_set:
|
||||
initialize_single_dummy_weight(tensor)
|
||||
|
||||
def _process_online_quant_layer(
|
||||
self,
|
||||
|
||||
@@ -1341,6 +1341,10 @@ def initialize_single_dummy_weight(
|
||||
if param.device.type == "meta":
|
||||
return # deferred to finalize_layerwise_processing (e.g. online quant)
|
||||
|
||||
if (dummy_weight_value := getattr(param, "dummy_weight_value", None)) is not None:
|
||||
param.fill_(dummy_weight_value)
|
||||
return
|
||||
|
||||
if not torch.is_floating_point(param):
|
||||
if current_platform.is_rocm():
|
||||
# On ROCm, integer params (e.g. GPTQ qweight/qzeros) are left
|
||||
|
||||
@@ -687,6 +687,10 @@ class ParallelEngramEmbedding(nn.Module):
|
||||
},
|
||||
)
|
||||
if cpu_offload:
|
||||
# Constant dummy values avoid randomizing huge CPU lookup tables.
|
||||
set_weight_attrs(self.weight, {"dummy_weight_value": 1.0})
|
||||
# The ue8m0 encoding of scale 1.0 is exponent byte 127.
|
||||
set_weight_attrs(self.weight_scale_inv, {"dummy_weight_value": 127})
|
||||
logger.info(
|
||||
"Engram table offloaded to pinned host memory: %d rows x %d, "
|
||||
"%.2f GiB per rank",
|
||||
|
||||
Reference in New Issue
Block a user