[QeRL] Fix online quantized reloading (#38442)

Signed-off-by: Kyle Sayers <kylesayrs@gmail.com>
This commit is contained in:
Kyle Sayers
2026-03-29 14:56:41 -06:00
committed by GitHub
parent 995dea1354
commit d28d86e8a3
9 changed files with 104 additions and 62 deletions
+17 -21
View File
@@ -4943,28 +4943,24 @@ class GPUModelRunner(
# begin loading weights
logger.info_once("Reloading weights inplace...", scope="local")
load_device = (
self.vllm_config.load_config.device or self.vllm_config.device_config.device
)
with torch.device(load_device):
if is_checkpoint_format:
# load weights from checkpoint/ original model format
initialize_layerwise_reload(model)
loaded_weights = model.load_weights(weights_iterator)
finalize_layerwise_reload(model, self.model_config)
if is_checkpoint_format:
# load weights from checkpoint/ original model format
initialize_layerwise_reload(model)
loaded_weights = model.load_weights(weights_iterator)
finalize_layerwise_reload(model, self.model_config)
else:
# load weights from kernel format
logger.warning_once(
"Reloading with `is_checkpoint_format=True` requires that "
"weights be in kernel format and already sharded",
scope="local",
)
loaded_weights = set()
for name, loaded_weight in weights_iterator:
param = model.get_parameter(name) # TODO: buffers?
param.copy_(loaded_weight)
loaded_weights.add(name)
else:
# load weights from kernel format
logger.warning_once(
"Reloading with `is_checkpoint_format=True` requires that "
"weights be in kernel format and already sharded",
scope="local",
)
loaded_weights = set()
for name, loaded_weight in weights_iterator:
param = model.get_parameter(name) # TODO: buffers?
param.copy_(loaded_weight)
loaded_weights.add(name)
# logging and validation
counter_after_reloading = time.perf_counter()