Convert formatting to use ruff instead of yapf + isort (#26247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -10,36 +10,45 @@ import torch.nn.functional as F
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (FusedMoE, FusedMoEConfig,
|
||||
FusedMoEMethodBase)
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEConfig,
|
||||
FusedMoEMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig, int4_w4a16_moe_quant_config,
|
||||
int8_w8a16_moe_quant_config)
|
||||
from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
|
||||
set_weight_attrs)
|
||||
FusedMoEQuantConfig,
|
||||
int4_w4a16_moe_quant_config,
|
||||
int8_w8a16_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
LinearBase,
|
||||
LinearMethodBase,
|
||||
set_weight_attrs,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationMethods
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig, QuantizeMethodBase)
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
"""By default, use 8 bit as target precision, but it can be
|
||||
overridden by setting the RTN_NUM_BITS envvar
|
||||
"""
|
||||
NUM_BITS = os.getenv('RTN_NUM_BITS', "8")
|
||||
NUM_BITS = os.getenv("RTN_NUM_BITS", "8")
|
||||
"""By default, use group size of 128 parameters, but it can be
|
||||
overridden by setting the RTN_GROUP_SIZE envvar
|
||||
"""
|
||||
GROUP_SIZE = os.getenv('RTN_GROUP_SIZE', "128")
|
||||
GROUP_SIZE = os.getenv("RTN_GROUP_SIZE", "128")
|
||||
|
||||
|
||||
class RTNConfig(QuantizationConfig):
|
||||
"""Config class for RTN.
|
||||
"""
|
||||
"""Config class for RTN."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
weight_bits: int = int(NUM_BITS),
|
||||
group_size: int = int(GROUP_SIZE),
|
||||
self,
|
||||
weight_bits: int = int(NUM_BITS),
|
||||
group_size: int = int(GROUP_SIZE),
|
||||
) -> None:
|
||||
self.weight_bits = weight_bits
|
||||
self.group_size = group_size
|
||||
@@ -47,11 +56,13 @@ class RTNConfig(QuantizationConfig):
|
||||
if self.weight_bits != 4 and self.weight_bits != 8:
|
||||
raise ValueError(
|
||||
"Currently, only 4-bit or 8-bit weight quantization is "
|
||||
f"supported for RTN, but got {self.weight_bits} bits.")
|
||||
f"supported for RTN, but got {self.weight_bits} bits."
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (f"RTNConfig(weight_bits={self.weight_bits}, "
|
||||
f"group_size={self.group_size})")
|
||||
return (
|
||||
f"RTNConfig(weight_bits={self.weight_bits}, group_size={self.group_size})"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> QuantizationMethods:
|
||||
@@ -75,8 +86,9 @@ class RTNConfig(QuantizationConfig):
|
||||
group_size = cls.get_from_keys(config, ["group_size"])
|
||||
return cls(weight_bits, group_size)
|
||||
|
||||
def get_quant_method(self, layer: torch.nn.Module,
|
||||
prefix: str) -> Optional["QuantizeMethodBase"]:
|
||||
def get_quant_method(
|
||||
self, layer: torch.nn.Module, prefix: str
|
||||
) -> Optional["QuantizeMethodBase"]:
|
||||
if isinstance(layer, LinearBase):
|
||||
return RTNLinearMethod(self)
|
||||
elif isinstance(layer, FusedMoE):
|
||||
@@ -89,8 +101,9 @@ class RTNTensor:
|
||||
overloading the copy_ method.
|
||||
"""
|
||||
|
||||
def __init__(self, data: torch.Tensor, scale: torch.Tensor,
|
||||
quant_config: RTNConfig) -> None:
|
||||
def __init__(
|
||||
self, data: torch.Tensor, scale: torch.Tensor, quant_config: RTNConfig
|
||||
) -> None:
|
||||
self.data = data
|
||||
self.scale = scale
|
||||
self.quant_config = quant_config
|
||||
@@ -99,7 +112,9 @@ class RTNTensor:
|
||||
factor = 1 if self.quant_config.weight_bits == 8 else 2
|
||||
return RTNTensor(
|
||||
self.data.narrow(dim, start // factor, length // factor),
|
||||
self.scale.narrow(dim, start, length), self.quant_config)
|
||||
self.scale.narrow(dim, start, length),
|
||||
self.quant_config,
|
||||
)
|
||||
|
||||
def __getitem__(self, key):
|
||||
return RTNTensor(self.data[key], self.scale[key], self.quant_config)
|
||||
@@ -115,9 +130,11 @@ class RTNTensor:
|
||||
return torch.Size((shape[0] * factor, shape[1]))
|
||||
|
||||
def copy_(self, loaded_weight: torch.Tensor) -> None:
|
||||
qweight, weight_scale = rtn_quantize(loaded_weight.cuda(),
|
||||
self.quant_config.weight_bits,
|
||||
self.quant_config.group_size)
|
||||
qweight, weight_scale = rtn_quantize(
|
||||
loaded_weight.cuda(),
|
||||
self.quant_config.weight_bits,
|
||||
self.quant_config.group_size,
|
||||
)
|
||||
|
||||
self.data.copy_(qweight)
|
||||
self.scale.data.copy_(weight_scale)
|
||||
@@ -133,8 +150,9 @@ class RTNParameter(Parameter):
|
||||
def __new__(cls, data: torch.Tensor, **kwargs):
|
||||
return super().__new__(cls, data=data, requires_grad=False)
|
||||
|
||||
def __init__(self, data: torch.Tensor, scale: torch.Tensor,
|
||||
quant_config: RTNConfig) -> None:
|
||||
def __init__(
|
||||
self, data: torch.Tensor, scale: torch.Tensor, quant_config: RTNConfig
|
||||
) -> None:
|
||||
self.scale = scale
|
||||
self.quant_config = quant_config
|
||||
|
||||
@@ -164,31 +182,39 @@ class RTNLinearMethod(LinearMethodBase):
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
num_groups_per_col = (input_size_per_partition //
|
||||
self.quant_config.group_size
|
||||
if self.quant_config.group_size != -1 else 1)
|
||||
num_groups_per_col = (
|
||||
input_size_per_partition // self.quant_config.group_size
|
||||
if self.quant_config.group_size != -1
|
||||
else 1
|
||||
)
|
||||
|
||||
scale = Parameter(
|
||||
torch.empty(output_size_per_partition,
|
||||
num_groups_per_col,
|
||||
dtype=params_dtype),
|
||||
torch.empty(
|
||||
output_size_per_partition, num_groups_per_col, dtype=params_dtype
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
factor = 1 if self.quant_config.weight_bits == 8 else 2
|
||||
|
||||
weight = RTNParameter(data=torch.empty(output_size_per_partition //
|
||||
factor,
|
||||
input_size_per_partition,
|
||||
dtype=torch.uint8),
|
||||
scale=scale,
|
||||
quant_config=self.quant_config)
|
||||
weight = RTNParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition // factor,
|
||||
input_size_per_partition,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
scale=scale,
|
||||
quant_config=self.quant_config,
|
||||
)
|
||||
|
||||
layer.register_parameter("weight", weight)
|
||||
set_weight_attrs(weight, {
|
||||
**extra_weight_attrs,
|
||||
"input_dim": 1,
|
||||
"output_dim": 0,
|
||||
})
|
||||
set_weight_attrs(
|
||||
weight,
|
||||
{
|
||||
**extra_weight_attrs,
|
||||
"input_dim": 1,
|
||||
"output_dim": 0,
|
||||
},
|
||||
)
|
||||
|
||||
layer.register_parameter("scale", scale)
|
||||
layer.output_size_per_partition = output_size_per_partition
|
||||
@@ -196,10 +222,12 @@ class RTNLinearMethod(LinearMethodBase):
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
fix_weights(layer, "weight")
|
||||
|
||||
def apply(self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
|
||||
def apply(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
qweight = layer.weight
|
||||
scale = layer.scale
|
||||
|
||||
@@ -213,57 +241,75 @@ class RTNLinearMethod(LinearMethodBase):
|
||||
|
||||
|
||||
class RTNMoEMethod(FusedMoEMethodBase):
|
||||
|
||||
def __init__(self, quant_config: RTNConfig, moe: FusedMoEConfig):
|
||||
super().__init__(moe)
|
||||
self.quant_config = quant_config
|
||||
|
||||
def create_weights(self, layer: torch.nn.Module, num_experts: int,
|
||||
hidden_size: int, intermediate_size_per_partition: int,
|
||||
params_dtype: torch.dtype, **extra_weight_attrs):
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
factor = 1 if self.quant_config.weight_bits == 8 else 2
|
||||
|
||||
# Fused gate_up_proj (column parallel)
|
||||
num_groups_per_col = (hidden_size // self.quant_config.group_size
|
||||
if self.quant_config.group_size != -1 else 1)
|
||||
num_groups_per_col = (
|
||||
hidden_size // self.quant_config.group_size
|
||||
if self.quant_config.group_size != -1
|
||||
else 1
|
||||
)
|
||||
w13_scale = Parameter(
|
||||
torch.empty(num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
num_groups_per_col,
|
||||
dtype=params_dtype),
|
||||
torch.empty(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
num_groups_per_col,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_scale", w13_scale)
|
||||
|
||||
w13_weight = RTNParameter(data=torch.empty(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition // factor,
|
||||
hidden_size,
|
||||
dtype=torch.uint8),
|
||||
scale=w13_scale,
|
||||
quant_config=self.quant_config)
|
||||
w13_weight = RTNParameter(
|
||||
data=torch.empty(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition // factor,
|
||||
hidden_size,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
scale=w13_scale,
|
||||
quant_config=self.quant_config,
|
||||
)
|
||||
layer.register_parameter("w13_weight", w13_weight)
|
||||
set_weight_attrs(w13_weight, extra_weight_attrs)
|
||||
|
||||
# down_proj (row parallel)
|
||||
num_groups_per_col = (intermediate_size_per_partition //
|
||||
self.quant_config.group_size
|
||||
if self.quant_config.group_size != -1 else 1)
|
||||
w2_scale = Parameter(torch.zeros(num_experts,
|
||||
hidden_size,
|
||||
num_groups_per_col,
|
||||
dtype=params_dtype),
|
||||
requires_grad=False)
|
||||
num_groups_per_col = (
|
||||
intermediate_size_per_partition // self.quant_config.group_size
|
||||
if self.quant_config.group_size != -1
|
||||
else 1
|
||||
)
|
||||
w2_scale = Parameter(
|
||||
torch.zeros(
|
||||
num_experts, hidden_size, num_groups_per_col, dtype=params_dtype
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_scale", w2_scale)
|
||||
|
||||
w2_weight = RTNParameter(data=torch.empty(
|
||||
num_experts,
|
||||
hidden_size // factor,
|
||||
intermediate_size_per_partition,
|
||||
dtype=torch.uint8),
|
||||
scale=w2_scale,
|
||||
quant_config=self.quant_config)
|
||||
w2_weight = RTNParameter(
|
||||
data=torch.empty(
|
||||
num_experts,
|
||||
hidden_size // factor,
|
||||
intermediate_size_per_partition,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
scale=w2_scale,
|
||||
quant_config=self.quant_config,
|
||||
)
|
||||
layer.register_parameter("w2_weight", w2_weight)
|
||||
set_weight_attrs(w2_weight, extra_weight_attrs)
|
||||
|
||||
@@ -273,12 +319,16 @@ class RTNMoEMethod(FusedMoEMethodBase):
|
||||
fix_weights(layer, "w2_weight", weight_bits == 4)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: torch.nn.Module) -> Optional[FusedMoEQuantConfig]:
|
||||
self, layer: torch.nn.Module
|
||||
) -> Optional[FusedMoEQuantConfig]:
|
||||
weight_bits = self.quant_config.weight_bits
|
||||
group_size = self.quant_config.group_size
|
||||
assert weight_bits == 4 or weight_bits == 8
|
||||
config_builder = (int4_w4a16_moe_quant_config
|
||||
if weight_bits == 4 else int8_w8a16_moe_quant_config)
|
||||
config_builder = (
|
||||
int4_w4a16_moe_quant_config
|
||||
if weight_bits == 4
|
||||
else int8_w8a16_moe_quant_config
|
||||
)
|
||||
return config_builder(
|
||||
w1_scale=layer.w13_scale,
|
||||
w2_scale=layer.w2_scale,
|
||||
@@ -313,8 +363,7 @@ class RTNMoEMethod(FusedMoEMethodBase):
|
||||
assert self.fused_experts is None
|
||||
|
||||
if enable_eplb:
|
||||
raise NotImplementedError(
|
||||
"EPLB not supported for `RTNMoEMethod` yet.")
|
||||
raise NotImplementedError("EPLB not supported for `RTNMoEMethod` yet.")
|
||||
|
||||
from vllm.model_executor.layers.fused_moe import fused_experts
|
||||
|
||||
@@ -330,7 +379,8 @@ class RTNMoEMethod(FusedMoEMethodBase):
|
||||
scoring_func=scoring_func,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
indices_type=self.topk_indices_dtype)
|
||||
indices_type=self.topk_indices_dtype,
|
||||
)
|
||||
|
||||
return fused_experts(
|
||||
x,
|
||||
@@ -347,15 +397,16 @@ class RTNMoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
|
||||
|
||||
def rtn_quantize(tensor: torch.Tensor, num_bits: int,
|
||||
group_size: int) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
def rtn_quantize(
|
||||
tensor: torch.Tensor, num_bits: int, group_size: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Quantize a tensor using per-group static scaling factor.
|
||||
|
||||
Args:
|
||||
tensor: The input tensor.
|
||||
num_bits: Target precision for the result (supported values are
|
||||
8 or 4).
|
||||
group_size: Quantization granularity.
|
||||
group_size: Quantization granularity.
|
||||
If equal to -1, each row in the input tensor is treated
|
||||
as one group.
|
||||
"""
|
||||
@@ -364,15 +415,18 @@ def rtn_quantize(tensor: torch.Tensor, num_bits: int,
|
||||
tensor = tensor.unsqueeze(0)
|
||||
|
||||
q_range = 2**num_bits
|
||||
num_groups = (tensor.shape[1] * tensor.shape[2] //
|
||||
group_size if group_size != -1 else tensor.shape[1])
|
||||
num_groups = (
|
||||
tensor.shape[1] * tensor.shape[2] // group_size
|
||||
if group_size != -1
|
||||
else tensor.shape[1]
|
||||
)
|
||||
"""Calculate a scaling factor per input group.
|
||||
"""
|
||||
input_flat = tensor.reshape(tensor.shape[0], num_groups, -1)
|
||||
input_min = torch.min(input_flat, dim=2, keepdim=True)[0]
|
||||
input_max = torch.max(input_flat, dim=2, keepdim=True)[0]
|
||||
input_max_abs = torch.max(input_min.abs(), input_max.abs())
|
||||
scale = (input_max_abs * 2.0 / (q_range - 1))
|
||||
scale = input_max_abs * 2.0 / (q_range - 1)
|
||||
"""Scale each input group, round to the nearest integer, shift
|
||||
the range and truncate.
|
||||
"""
|
||||
@@ -388,9 +442,10 @@ def rtn_quantize(tensor: torch.Tensor, num_bits: int,
|
||||
if num_bits == 4:
|
||||
"""Pack two 4-bit values into each byte.
|
||||
"""
|
||||
inputs_q = (inputs_q[:, :, 1::2] << 4) | (inputs_q[:, :, ::2] & 0xf)
|
||||
inputs_q = inputs_q.reshape(tensor.shape[0], tensor.shape[1] // 2,
|
||||
tensor.shape[2])
|
||||
inputs_q = (inputs_q[:, :, 1::2] << 4) | (inputs_q[:, :, ::2] & 0xF)
|
||||
inputs_q = inputs_q.reshape(
|
||||
tensor.shape[0], tensor.shape[1] // 2, tensor.shape[2]
|
||||
)
|
||||
inputs_q = inputs_q.contiguous()
|
||||
|
||||
if not batch_present:
|
||||
@@ -420,9 +475,9 @@ def rtn_dequantize(tensor: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
|
||||
if num_bits == 4:
|
||||
input_dim *= 2
|
||||
|
||||
data = torch.empty((batch, input_dim, output_dim),
|
||||
dtype=scale.dtype,
|
||||
device=tensor.device)
|
||||
data = torch.empty(
|
||||
(batch, input_dim, output_dim), dtype=scale.dtype, device=tensor.device
|
||||
)
|
||||
|
||||
if num_bits == 8:
|
||||
data.copy_(tensor)
|
||||
@@ -432,8 +487,9 @@ def rtn_dequantize(tensor: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
tensor = tensor.reshape(batch, input_dim, output_dim // 2)
|
||||
for i in range(2):
|
||||
data[:, :, i::2] = ((tensor << 4 *
|
||||
(1 - i)) >> 4).to(torch.int8) - q_range // 2
|
||||
data[:, :, i::2] = ((tensor << 4 * (1 - i)) >> 4).to(
|
||||
torch.int8
|
||||
) - q_range // 2
|
||||
"""Scale each input group with its scaling factor.
|
||||
"""
|
||||
scale = scale.reshape(batch, num_groups, -1)
|
||||
@@ -447,9 +503,7 @@ def rtn_dequantize(tensor: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
|
||||
return input_deq
|
||||
|
||||
|
||||
def fix_weights(layer: torch.nn.Module,
|
||||
param_name: str,
|
||||
reshape: bool = False):
|
||||
def fix_weights(layer: torch.nn.Module, param_name: str, reshape: bool = False):
|
||||
"""torch.compile does not know how to deal with a Parameter subclass
|
||||
(aka RTNParameter). As we don't really need RTNParameters for the
|
||||
forward pass, we replace them with equivalent instances of Parameters.
|
||||
|
||||
Reference in New Issue
Block a user