Support RL online quantization with torchao (#23014)

Signed-off-by: Jerry Zhang <jerryzh168@gmail.com>
This commit is contained in:
Jerry Zhang
2025-10-01 16:39:29 -07:00
committed by GitHub
parent 4134312b35
commit c31246800c
6 changed files with 465 additions and 16 deletions
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from typing import Any, Optional
import torch
@@ -40,7 +41,8 @@ class TorchAOConfig(QuantizationConfig):
def __init__(self,
torchao_config,
skip_modules: Optional[list[str]] = None) -> None:
skip_modules: Optional[list[str]] = None,
is_checkpoint_torchao_serialized: bool = False) -> None:
"""
# TorchAO quantization relies on tensor subclasses. In order,
# to enable proper caching this needs standalone compile
@@ -58,9 +60,11 @@ class TorchAOConfig(QuantizationConfig):
super().__init__()
self.torchao_config = torchao_config
self.skip_modules = skip_modules or []
self.is_checkpoint_torchao_serialized = is_checkpoint_torchao_serialized
def __repr__(self) -> str:
return f"TorchAOConfig({self.torchao_config})"
return f"TorchAOConfig({self.torchao_config=}, {self.skip_modules=}, " \
f"{self.is_checkpoint_torchao_serialized=})"
def get_name(self) -> QuantizationMethods:
return "torchao"
@@ -74,7 +78,10 @@ class TorchAOConfig(QuantizationConfig):
@staticmethod
def get_config_filenames() -> list[str]:
return ["config.json"]
"""torchao doesn't require additional config files, we use
`config.json` from huggingface: `model_config.hf_config`
"""
return []
@classmethod
def from_config(cls, config: dict[str, Any]) -> "TorchAOConfig":
@@ -87,6 +94,10 @@ class TorchAOConfig(QuantizationConfig):
"`pip install torchao>=0.10.0` to use torchao quantization."
) from err
quant_method = cls.get_from_keys_or(config, ["quant_method"], None)
is_checkpoint_torchao_serialized = (quant_method is not None
and "torchao" in quant_method)
hf_config = cls.get_from_keys_or(config, ["quant_type"], None)
assert hf_config is not None, "quant_type must be specified"
assert len(hf_config) == 1 and "default" in hf_config, (
@@ -110,7 +121,38 @@ class TorchAOConfig(QuantizationConfig):
if layer_cfg is None:
skip_modules.append(layer)
return cls(ao_config, skip_modules)
return cls(ao_config, skip_modules, is_checkpoint_torchao_serialized)
@classmethod
def from_config_file(cls, config_file: str) -> "TorchAOConfig":
"""Initialize class from a config file. Example:
```
config = (
Float8DynamicActivationFloat8WeightConfig(granularity=PerRow())
)
fn = "torchao_config.json"
with open(fn, "w") as f:
f.write(json.dumps(config_to_dict(config)))
```
"""
with open(config_file) as f:
f.seek(0)
f_read = f.read()
config_dict = json.loads(f_read)
hf_config = {"quant_type": {"default": config_dict}}
return cls.from_config(hf_config)
@classmethod
def from_config_dict_json(cls, config_dict_json: str) -> "TorchAOConfig":
"""Iniitalize class from a config_dict json string, got from
torchao_config_object = some AOBaseConfig object
json.dumps(config_to_dict(torchao_config_object))
"""
config_dict = json.loads(config_dict_json)
hf_config = {"quant_type": {"default": config_dict}}
return cls.from_config(hf_config)
def get_quant_method(self, layer: torch.nn.Module,
prefix: str) -> Optional["QuantizeMethodBase"]:
@@ -128,7 +170,9 @@ class TorchAOConfig(QuantizationConfig):
c = module_fqn_to_config.get(
module_fqn) or module_fqn_to_config.get("_default", None)
if c is not None:
current_torchao_config = TorchAOConfig(c, self.skip_modules)
current_torchao_config = TorchAOConfig(
c, self.skip_modules,
self.is_checkpoint_torchao_serialized)
return TorchAOLinearMethod(current_torchao_config)
else:
return UnquantizedLinearMethod()
@@ -172,7 +216,7 @@ class TorchAOLinearMethod(LinearMethodBase):
"""Linear method for torchao.
Args:
quant_config: The torchao quantization config, a string that encodes
quant_config: The torchao quantization config, a string that encodes
the type of quantization and all relevant arguments.
"""
@@ -197,8 +241,9 @@ class TorchAOLinearMethod(LinearMethodBase):
),
requires_grad=False,
)
weight = torchao_quantize_param_data(weight,
self.quant_config.torchao_config)
if self.quant_config.is_checkpoint_torchao_serialized:
weight = torchao_quantize_param_data(
weight, self.quant_config.torchao_config)
set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
@@ -212,3 +257,14 @@ class TorchAOLinearMethod(LinearMethodBase):
bias: Optional[torch.Tensor] = None,
) -> torch.Tensor:
return F.linear(x, layer.weight, bias)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
if self.quant_config.is_checkpoint_torchao_serialized:
return
# quantize the weight on the fly if the checkpoint is not already
# quantized by torchao
weight = torchao_quantize_param_data(layer.weight,
self.quant_config.torchao_config)
set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
layer.register_parameter("weight", weight)