[Quantization] - Added uses_meta_device_weights to quant config (#34645)
Signed-off-by: Josephasafg <ajgard7@gmail.com>
This commit is contained in:
@@ -18,6 +18,11 @@ else:
|
||||
class QuantizeMethodBase(ABC):
|
||||
"""Base class for different quantized methods."""
|
||||
|
||||
# Whether this method creates weights on meta device for online quantization.
|
||||
# When True, weights are created on meta device and quantized layer-wise
|
||||
# in process_weights_after_loading, reducing peak memory during loading.
|
||||
uses_meta_device: bool = False
|
||||
|
||||
@abstractmethod
|
||||
def create_weights(
|
||||
self, layer: torch.nn.Module, *weight_args, **extra_weight_attrs
|
||||
|
||||
@@ -527,6 +527,8 @@ class Fp8OnlineLinearMethod(Fp8LinearMethod):
|
||||
"""Online version of Fp8LinearMethod, loads the fp16/bf16 checkpoint
|
||||
and quantized the weights during loading."""
|
||||
|
||||
uses_meta_device: bool = True
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
@@ -1039,6 +1041,8 @@ class Fp8OnlineMoEMethod(Fp8MoEMethod):
|
||||
quant_config: The quantization config.
|
||||
"""
|
||||
|
||||
uses_meta_device: bool = True
|
||||
|
||||
def __init__(self, quant_config: Fp8Config, layer: torch.nn.Module):
|
||||
super().__init__(quant_config, layer)
|
||||
assert not quant_config.is_checkpoint_fp8_serialized
|
||||
|
||||
Reference in New Issue
Block a user