Convert formatting to use ruff instead of yapf + isort (#26247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -8,11 +8,16 @@ import torch.nn.functional as F
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
|
||||
UnquantizedLinearMethod)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
LinearBase,
|
||||
LinearMethodBase,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationMethods
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig, QuantizeMethodBase)
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -39,10 +44,12 @@ def should_skip(prefix: str, skip_modules: list[str]) -> bool:
|
||||
class TorchAOConfig(QuantizationConfig):
|
||||
"""Config class for torchao."""
|
||||
|
||||
def __init__(self,
|
||||
torchao_config,
|
||||
skip_modules: Optional[list[str]] = None,
|
||||
is_checkpoint_torchao_serialized: bool = False) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
torchao_config,
|
||||
skip_modules: Optional[list[str]] = None,
|
||||
is_checkpoint_torchao_serialized: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
# TorchAO quantization relies on tensor subclasses. In order,
|
||||
# to enable proper caching this needs standalone compile
|
||||
@@ -63,8 +70,10 @@ class TorchAOConfig(QuantizationConfig):
|
||||
self.is_checkpoint_torchao_serialized = is_checkpoint_torchao_serialized
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"TorchAOConfig({self.torchao_config=}, {self.skip_modules=}, " \
|
||||
return (
|
||||
f"TorchAOConfig({self.torchao_config=}, {self.skip_modules=}, "
|
||||
f"{self.is_checkpoint_torchao_serialized=})"
|
||||
)
|
||||
|
||||
def get_name(self) -> QuantizationMethods:
|
||||
return "torchao"
|
||||
@@ -95,13 +104,15 @@ class TorchAOConfig(QuantizationConfig):
|
||||
) from err
|
||||
|
||||
quant_method = cls.get_from_keys_or(config, ["quant_method"], None)
|
||||
is_checkpoint_torchao_serialized = (quant_method is not None
|
||||
and "torchao" in quant_method)
|
||||
is_checkpoint_torchao_serialized = (
|
||||
quant_method is not None and "torchao" in quant_method
|
||||
)
|
||||
|
||||
hf_config = cls.get_from_keys_or(config, ["quant_type"], None)
|
||||
assert hf_config is not None, "quant_type must be specified"
|
||||
assert len(hf_config) == 1 and "default" in hf_config, (
|
||||
"Expected only one key 'default' in quant_type dictionary")
|
||||
"Expected only one key 'default' in quant_type dictionary"
|
||||
)
|
||||
quant_type = hf_config["default"]
|
||||
ao_config = config_from_dict(quant_type)
|
||||
|
||||
@@ -127,9 +138,7 @@ class TorchAOConfig(QuantizationConfig):
|
||||
def from_config_file(cls, config_file: str) -> "TorchAOConfig":
|
||||
"""Initialize class from a config file. Example:
|
||||
```
|
||||
config = (
|
||||
Float8DynamicActivationFloat8WeightConfig(granularity=PerRow())
|
||||
)
|
||||
config = Float8DynamicActivationFloat8WeightConfig(granularity=PerRow())
|
||||
fn = "torchao_config.json"
|
||||
|
||||
with open(fn, "w") as f:
|
||||
@@ -154,8 +163,9 @@ class TorchAOConfig(QuantizationConfig):
|
||||
hf_config = {"quant_type": {"default": config_dict}}
|
||||
return cls.from_config(hf_config)
|
||||
|
||||
def get_quant_method(self, layer: torch.nn.Module,
|
||||
prefix: str) -> Optional["QuantizeMethodBase"]:
|
||||
def get_quant_method(
|
||||
self, layer: torch.nn.Module, prefix: str
|
||||
) -> Optional["QuantizeMethodBase"]:
|
||||
if not isinstance(layer, LinearBase):
|
||||
return None
|
||||
|
||||
@@ -167,12 +177,13 @@ class TorchAOConfig(QuantizationConfig):
|
||||
module_fqn = prefix
|
||||
if isinstance(self.torchao_config, ModuleFqnToConfig):
|
||||
module_fqn_to_config = self.torchao_config.module_fqn_to_config
|
||||
c = module_fqn_to_config.get(
|
||||
module_fqn) or module_fqn_to_config.get("_default", None)
|
||||
c = module_fqn_to_config.get(module_fqn) or module_fqn_to_config.get(
|
||||
"_default", None
|
||||
)
|
||||
if c is not None:
|
||||
current_torchao_config = TorchAOConfig(
|
||||
c, self.skip_modules,
|
||||
self.is_checkpoint_torchao_serialized)
|
||||
c, self.skip_modules, self.is_checkpoint_torchao_serialized
|
||||
)
|
||||
return TorchAOLinearMethod(current_torchao_config)
|
||||
else:
|
||||
return UnquantizedLinearMethod()
|
||||
@@ -183,8 +194,9 @@ class TorchAOConfig(QuantizationConfig):
|
||||
return []
|
||||
|
||||
|
||||
def torchao_quantize_param_data(param: torch.Tensor,
|
||||
torchao_config: Any) -> torch.nn.Parameter:
|
||||
def torchao_quantize_param_data(
|
||||
param: torch.Tensor, torchao_config: Any
|
||||
) -> torch.nn.Parameter:
|
||||
"""Quantize a Tensor with torchao quantization specified by torchao_config
|
||||
|
||||
Args:
|
||||
@@ -205,7 +217,8 @@ def torchao_quantize_param_data(param: torch.Tensor,
|
||||
# while some of our configs need to do module swap, and only non-top
|
||||
# level modules support module swap
|
||||
dummy_linear = torch.nn.Sequential(
|
||||
torch.nn.Linear(param.shape[1], param.shape[0], bias=False))
|
||||
torch.nn.Linear(param.shape[1], param.shape[0], bias=False)
|
||||
)
|
||||
|
||||
dummy_linear[0].weight = param
|
||||
quantize_(dummy_linear, torchao_config)
|
||||
@@ -243,7 +256,8 @@ class TorchAOLinearMethod(LinearMethodBase):
|
||||
)
|
||||
if self.quant_config.is_checkpoint_torchao_serialized:
|
||||
weight = torchao_quantize_param_data(
|
||||
weight, self.quant_config.torchao_config)
|
||||
weight, self.quant_config.torchao_config
|
||||
)
|
||||
|
||||
set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
|
||||
|
||||
@@ -264,7 +278,8 @@ class TorchAOLinearMethod(LinearMethodBase):
|
||||
|
||||
# quantize the weight on the fly if the checkpoint is not already
|
||||
# quantized by torchao
|
||||
weight = torchao_quantize_param_data(layer.weight,
|
||||
self.quant_config.torchao_config)
|
||||
weight = torchao_quantize_param_data(
|
||||
layer.weight, self.quant_config.torchao_config
|
||||
)
|
||||
set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
|
||||
layer.register_parameter("weight", weight)
|
||||
|
||||
Reference in New Issue
Block a user