[QeRL] Fix online quantized reloading (#38442)
Signed-off-by: Kyle Sayers <kylesayrs@gmail.com>
This commit is contained in:
@@ -4943,28 +4943,24 @@ class GPUModelRunner(
|
||||
|
||||
# begin loading weights
|
||||
logger.info_once("Reloading weights inplace...", scope="local")
|
||||
load_device = (
|
||||
self.vllm_config.load_config.device or self.vllm_config.device_config.device
|
||||
)
|
||||
with torch.device(load_device):
|
||||
if is_checkpoint_format:
|
||||
# load weights from checkpoint/ original model format
|
||||
initialize_layerwise_reload(model)
|
||||
loaded_weights = model.load_weights(weights_iterator)
|
||||
finalize_layerwise_reload(model, self.model_config)
|
||||
if is_checkpoint_format:
|
||||
# load weights from checkpoint/ original model format
|
||||
initialize_layerwise_reload(model)
|
||||
loaded_weights = model.load_weights(weights_iterator)
|
||||
finalize_layerwise_reload(model, self.model_config)
|
||||
|
||||
else:
|
||||
# load weights from kernel format
|
||||
logger.warning_once(
|
||||
"Reloading with `is_checkpoint_format=True` requires that "
|
||||
"weights be in kernel format and already sharded",
|
||||
scope="local",
|
||||
)
|
||||
loaded_weights = set()
|
||||
for name, loaded_weight in weights_iterator:
|
||||
param = model.get_parameter(name) # TODO: buffers?
|
||||
param.copy_(loaded_weight)
|
||||
loaded_weights.add(name)
|
||||
else:
|
||||
# load weights from kernel format
|
||||
logger.warning_once(
|
||||
"Reloading with `is_checkpoint_format=True` requires that "
|
||||
"weights be in kernel format and already sharded",
|
||||
scope="local",
|
||||
)
|
||||
loaded_weights = set()
|
||||
for name, loaded_weight in weights_iterator:
|
||||
param = model.get_parameter(name) # TODO: buffers?
|
||||
param.copy_(loaded_weight)
|
||||
loaded_weights.add(name)
|
||||
|
||||
# logging and validation
|
||||
counter_after_reloading = time.perf_counter()
|
||||
|
||||
Reference in New Issue
Block a user