[QeRL] Fix online quantized reloading (#38442)
Signed-off-by: Kyle Sayers <kylesayrs@gmail.com>
This commit is contained in:
@@ -1006,14 +1006,17 @@ class Fp8OnlineMoEMethod(Fp8MoEMethod):
|
||||
initialize_online_processing(layer)
|
||||
|
||||
def process_weights_after_loading(self, layer: Module) -> None:
|
||||
# TODO(@ksayers): inplace fp8 quant kernel, initialize scales with ones
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
return
|
||||
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
w13 = torch.empty_like(layer.w13_weight, dtype=fp8_dtype)
|
||||
w2 = torch.empty_like(layer.w2_weight, dtype=fp8_dtype)
|
||||
w13_scale = torch.ones(layer.num_experts, dtype=torch.float32)
|
||||
w2_scale = torch.ones(layer.num_experts, dtype=torch.float32)
|
||||
w13_scale = torch.ones(
|
||||
layer.num_experts, device=w13.device, dtype=torch.float32
|
||||
)
|
||||
w2_scale = torch.ones(layer.num_experts, device=w2.device, dtype=torch.float32)
|
||||
layer.w13_input_scale = None
|
||||
layer.w2_input_scale = None
|
||||
|
||||
|
||||
@@ -49,7 +49,10 @@ def get_layerwise_info(layer: torch.nn.Module) -> LayerReloadingInfo:
|
||||
information existed, a new entry is constructed
|
||||
"""
|
||||
if layer not in LAYERWISE_INFO:
|
||||
LAYERWISE_INFO[layer] = LayerReloadingInfo()
|
||||
LAYERWISE_INFO[layer] = LayerReloadingInfo(
|
||||
restore_metadata=({}, {}),
|
||||
restore_device=torch.get_default_device(),
|
||||
)
|
||||
|
||||
return LAYERWISE_INFO[layer]
|
||||
|
||||
@@ -64,6 +67,7 @@ def record_metadata_for_reloading(model: torch.nn.Module):
|
||||
for layer in model.modules():
|
||||
info = get_layerwise_info(layer)
|
||||
info.restore_metadata = capture_layer_to_meta(layer)
|
||||
info.restore_device = torch.get_default_device()
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
@@ -99,10 +103,18 @@ def initialize_layerwise_reload(model: torch.nn.Module):
|
||||
# Restore layer parameters/buffers onto meta device
|
||||
restore_layer_on_meta(layer, info)
|
||||
|
||||
# Wrap weight loaders to buffer loading
|
||||
initialize_online_processing(layer)
|
||||
|
||||
|
||||
def initialize_online_processing(layer: torch.nn.Module):
|
||||
"""
|
||||
Wrap a layer's weight loaders with online processing loaders.
|
||||
Called by either `initialize_layerwise_reload` or an online quantization scheme,
|
||||
prevents double wrapping in the case of online quantization + reloading
|
||||
|
||||
:param layer: layer whose parameter weight loaders will be wrapped
|
||||
"""
|
||||
info = get_layerwise_info(layer)
|
||||
|
||||
# Track loading progress to determine when to process/copy
|
||||
@@ -211,7 +223,7 @@ def finalize_layerwise_processing(model: torch.nn.Module, model_config: ModelCon
|
||||
elif info.load_numel <= 0:
|
||||
# first load but received no weights. This happens on dummy load
|
||||
if info.kernel_tensors is None:
|
||||
materialize_layer(layer)
|
||||
materialize_layer(layer, info)
|
||||
|
||||
# reloading: place kernel tensors back as a fallback
|
||||
else:
|
||||
@@ -244,7 +256,7 @@ def _layerwise_process(layer: torch.nn.Module, info: LayerReloadingInfo):
|
||||
4. Copies processed values back to original tensor storage
|
||||
"""
|
||||
# Materialize layer tensors onto device
|
||||
materialize_layer(layer)
|
||||
materialize_layer(layer, info)
|
||||
|
||||
# Reset online quantization flag so process_weights_after_loading
|
||||
# will run again during reload
|
||||
|
||||
@@ -94,14 +94,15 @@ def restore_layer_on_meta(layer: torch.nn.Module, info: LayerReloadingInfo):
|
||||
layer.register_buffer(name, buffer)
|
||||
|
||||
|
||||
def materialize_layer(layer: torch.nn.Module) -> None:
|
||||
def materialize_layer(layer: torch.nn.Module, info: LayerReloadingInfo):
|
||||
"""Materialize all meta tensors in a layer to actual tensors."""
|
||||
if layer.__class__.__name__ in SKIP_MODULES:
|
||||
return
|
||||
|
||||
for name, tensor in get_layer_tensors(layer).items():
|
||||
if name not in SKIP_TENSORS:
|
||||
setattr(layer, name, materialize_meta_tensor(tensor))
|
||||
with info.restore_device:
|
||||
for name, tensor in get_layer_tensors(layer).items():
|
||||
if name not in SKIP_TENSORS:
|
||||
setattr(layer, name, materialize_meta_tensor(tensor))
|
||||
|
||||
|
||||
class CopyCounter(TorchDispatchMode):
|
||||
|
||||
@@ -13,21 +13,26 @@ LayerTensors = tuple[dict[str, torch.Tensor], dict[str, torch.Tensor]]
|
||||
|
||||
@dataclass
|
||||
class LayerReloadingInfo:
|
||||
# model format (meta), populated by `record_metadata_for_reloading`
|
||||
restore_metadata: LayerTensors = field(default_factory=lambda: ({}, {}))
|
||||
# model format metadata, recorded by `record_metadata_for_reloading`
|
||||
restore_metadata: LayerTensors
|
||||
|
||||
# kernel format (device), used to copy into when reloading only
|
||||
kernel_tensors: LayerTensors | None = None
|
||||
# device to materialize layers with, recorded by `record_metadata_for_reloading`
|
||||
restore_device: torch.device
|
||||
|
||||
# track how many restored elements are ready for loading
|
||||
# track how many elements are ready for loading, used by `online_process_loader`
|
||||
load_numel: int = 0
|
||||
load_numel_total: int | None = None
|
||||
|
||||
# stores arguments and tensors ready for loading
|
||||
# used by `online_process_loader` to buffer args and tensors until ready to load
|
||||
loaded_weights: list[tuple[str, BoundArguments]] = field(default_factory=list)
|
||||
|
||||
# kernel formatted tensors, copied into by `_layerwise_process` when reloading
|
||||
kernel_tensors: LayerTensors | None = None
|
||||
|
||||
def reset(self):
|
||||
self.__init__(restore_metadata=self.restore_metadata) # type: ignore[misc]
|
||||
self.__init__( # type: ignore[misc]
|
||||
restore_metadata=self.restore_metadata, restore_device=self.restore_device
|
||||
)
|
||||
|
||||
def can_load(self) -> bool:
|
||||
return self.load_numel_total is not None
|
||||
|
||||
Reference in New Issue
Block a user