[Bugfix] Re-sync parameter tp_rank after process_weights_after_loading (fix replicated / disable_tp weight reload) (#48025)
Signed-off-by: Alex Xu <alexxu@roblox.com> Co-authored-by: YQ-Wang <yiqingwang@roblox.com> Co-authored-by: alexhxu <alex.xu1015@gmail.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
YQ-Wang
alexhxu
Cursor
parent
c71a583aa9
commit
d96aee0951
@@ -364,6 +364,11 @@ def _layerwise_process(layer: torch.nn.Module, info: LayerReloadingInfo):
|
||||
quant_method = getattr(layer, "quant_method", None)
|
||||
if isinstance(quant_method, QuantizeMethodBase):
|
||||
quant_method.process_weights_after_loading(layer)
|
||||
# Re-reconcile parameter TP state: process_weights_after_loading may
|
||||
# have re-created Parameters (stamped with the global rank), which would
|
||||
# otherwise break replicated (disable_tp) weights on a subsequent reload.
|
||||
if hasattr(layer, "update_param_tp_status"):
|
||||
layer.update_param_tp_status()
|
||||
|
||||
# Copy processed values into original tensor storage (preserves cudagraph refs)
|
||||
# this code is a no-op if not reloading (because kernel tensors is empty)
|
||||
|
||||
@@ -111,6 +111,13 @@ def process_weights_after_loading(
|
||||
# parameters onto device for processing and back off after.
|
||||
with device_loading_context(module, target_device):
|
||||
quant_method.process_weights_after_loading(module)
|
||||
# process_weights_after_loading may swap in freshly-created
|
||||
# Parameters (e.g. FP8 requantization), which are stamped with the
|
||||
# global rank in BasevLLMParameter.__init__. Re-reconcile their TP
|
||||
# state to the layer so a later weight reload / RL weight-refit
|
||||
# narrows replicated (disable_tp) weights at the correct offset.
|
||||
if hasattr(module, "update_param_tp_status"):
|
||||
module.update_param_tp_status()
|
||||
# Repacking transients above can leave large amounts of memory in
|
||||
# the caching allocator, which starves the OS on UMA devices.
|
||||
release_device_memory_under_pressure(target_device)
|
||||
|
||||
Reference in New Issue
Block a user