[Quantization] - Added uses_meta_device_weights to quant config (#34645)

Signed-off-by: Josephasafg <ajgard7@gmail.com>
This commit is contained in:
Asaf Joseph Gardin
2026-02-17 23:43:44 -08:00
committed by GitHub
parent e89a91d927
commit 1faa8cb73c
3 changed files with 21 additions and 8 deletions
@@ -18,6 +18,11 @@ else:
class QuantizeMethodBase(ABC):
"""Base class for different quantized methods."""
# Whether this method creates weights on meta device for online quantization.
# When True, weights are created on meta device and quantized layer-wise
# in process_weights_after_loading, reducing peak memory during loading.
uses_meta_device: bool = False
@abstractmethod
def create_weights(
self, layer: torch.nn.Module, *weight_args, **extra_weight_attrs
@@ -527,6 +527,8 @@ class Fp8OnlineLinearMethod(Fp8LinearMethod):
"""Online version of Fp8LinearMethod, loads the fp16/bf16 checkpoint
and quantized the weights during loading."""
uses_meta_device: bool = True
def create_weights(
self,
layer: torch.nn.Module,
@@ -1039,6 +1041,8 @@ class Fp8OnlineMoEMethod(Fp8MoEMethod):
quant_config: The quantization config.
"""
uses_meta_device: bool = True
def __init__(self, quant_config: Fp8Config, layer: torch.nn.Module):
super().__init__(quant_config, layer)
assert not quant_config.is_checkpoint_fp8_serialized