[QeRL] Compose online quantization with quantized reloading (#38032)
Signed-off-by: Kyle Sayers <kylesayrs@gmail.com>
This commit is contained in:
@@ -1323,25 +1323,11 @@ def initialize_dummy_weights(
|
||||
is fixed, the random values generated by this function only depends on
|
||||
the parameter's number of elements and its data type.
|
||||
"""
|
||||
|
||||
# Check if any module uses online quantization with meta device weights.
|
||||
# If so, we'll skip initializing params on meta device since they'll be
|
||||
# handled in `process_weights_after_loading`.
|
||||
def uses_meta_device(module: torch.nn.Module) -> bool:
|
||||
quant_method = getattr(module, "quant_method", None)
|
||||
return getattr(quant_method, "uses_meta_device", False)
|
||||
|
||||
has_online_quant = any(uses_meta_device(m) for m in model.modules())
|
||||
|
||||
for param in model.state_dict().values():
|
||||
if has_online_quant and param.device == torch.device("meta"):
|
||||
# For online quantization, weights are created on meta device and
|
||||
# dummy weight init will happen in `process_weights_after_loading`.
|
||||
continue
|
||||
|
||||
initialize_single_dummy_weight(param, low, high, seed)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def initialize_single_dummy_weight(
|
||||
param: torch.Tensor,
|
||||
low: float = -1e-3,
|
||||
|
||||
Reference in New Issue
Block a user