[Quant] Support compressed-tensors WNA8O8Int linears and WNInt embeddings (#44340)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import regex as re
|
||||
@@ -42,16 +43,57 @@ def humming_is_layer_skipped(config: dict[str, Any], prefix: str):
|
||||
return False
|
||||
|
||||
|
||||
def convert_linear_layer_to_humming_standard(
|
||||
layer: LinearBase, name_map: dict[str, str]
|
||||
):
|
||||
"""Rename/reshape a linear layer's quantized params (the canonical MPLinear
|
||||
layout: ``weight_packed`` int32 + ``weight_scale``) into the parameter names
|
||||
and layout humming's weight schema expects (``weight`` / ``weight_scale``)."""
|
||||
for name, checkpoint_name in name_map.items():
|
||||
tensor = getattr(layer, checkpoint_name)
|
||||
delattr(layer, checkpoint_name)
|
||||
|
||||
if name == "weight":
|
||||
input_dim = getattr(tensor, "input_dim", 1)
|
||||
output_dim = getattr(tensor, "output_dim", 0)
|
||||
|
||||
if input_dim == 0 and output_dim == 1:
|
||||
tensor = tensor.transpose(1, 0).contiguous()
|
||||
else:
|
||||
assert output_dim == 0 and input_dim == 1
|
||||
|
||||
tensor = tensor.view(tensor.size(0), -1).view(torch.int32)
|
||||
elif name in ["weight_scale", "zero_point"]:
|
||||
if getattr(tensor, "output_dim", 0) == 1:
|
||||
tensor = tensor.transpose(0, 1).contiguous()
|
||||
if tensor.ndim == 1:
|
||||
tensor = tensor.unsqueeze(1)
|
||||
|
||||
tensor = tensor.view(torch.int32) if name == "zero_point" else tensor
|
||||
|
||||
if isinstance(tensor, torch.nn.Parameter):
|
||||
param = tensor
|
||||
else:
|
||||
param = torch.nn.Parameter(tensor, requires_grad=False)
|
||||
|
||||
setattr(layer, name, param)
|
||||
|
||||
|
||||
def prepare_humming_layer(layer: LinearBase, quant_config: dict):
|
||||
weight_schema = BaseWeightSchema.from_config(quant_config)
|
||||
input_schema = HummingInputSchema()
|
||||
|
||||
shape_k_stacks = [layer.input_size_per_partition]
|
||||
# ReplicatedLinear has no TP partitioning and so does not set
|
||||
# input_size_per_partition; for it that is just input_size.
|
||||
input_size_per_partition = getattr(
|
||||
layer, "input_size_per_partition", layer.input_size
|
||||
)
|
||||
shape_k_stacks = [input_size_per_partition]
|
||||
shape_n_stacks = layer.output_partition_sizes
|
||||
|
||||
# Step 1: convert weight to humming standard format
|
||||
weight_schema, tensors = weight_schema.convert_humming(
|
||||
tensors=layer.named_parameters(),
|
||||
tensors=dict(layer.named_parameters()),
|
||||
shape_n_stacks=shape_n_stacks,
|
||||
shape_k_stacks=shape_k_stacks,
|
||||
param_dtype=layer.params_dtype,
|
||||
@@ -63,23 +105,37 @@ def prepare_humming_layer(layer: LinearBase, quant_config: dict):
|
||||
delattr(layer, name)
|
||||
|
||||
for name, tensor in tensors.items():
|
||||
if isinstance(tensor, torch.nn.Parameter):
|
||||
tensor = tensor.data
|
||||
param = torch.nn.Parameter(tensor, requires_grad=False)
|
||||
setattr(layer, name, param)
|
||||
|
||||
# Step 2: transform weight (humming standard format) for forwarding
|
||||
HummingMethod.prepare_layer_meta(
|
||||
layer=layer,
|
||||
shape_n=layer.output_partition_sizes_sum,
|
||||
shape_k=layer.input_size_per_partition,
|
||||
shape_n=sum(layer.output_partition_sizes),
|
||||
shape_k=input_size_per_partition,
|
||||
weight_schema=weight_schema,
|
||||
input_schema=input_schema,
|
||||
pad_n_to_multiple=256,
|
||||
pad_k_to_multiple=128,
|
||||
has_bias=layer.has_bias,
|
||||
torch_dtype=layer.param_dtype,
|
||||
torch_dtype=layer.params_dtype,
|
||||
)
|
||||
|
||||
HummingMethod.transform_humming_layer(layer)
|
||||
if not hasattr(layer, "locks"):
|
||||
device = layer.weight.device
|
||||
locks = torch.zeros(1024, dtype=torch.int32, device=device)
|
||||
layer.register_buffer("locks", locks)
|
||||
|
||||
compute_config = {
|
||||
"use_batch_invariant": envs.VLLM_BATCH_INVARIANT,
|
||||
"use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM,
|
||||
"gemm_type": "dense",
|
||||
}
|
||||
|
||||
layer.compute_config = json.dumps(compute_config)
|
||||
|
||||
|
||||
def prepare_humming_moe_layer(layer: RoutedExperts, quant_config: dict):
|
||||
|
||||
Reference in New Issue
Block a user