[Quant] Support compressed-tensors WNA8O8Int linears and WNInt embeddings (#44340)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Michael Goin
2026-06-04 07:40:33 -07:00
committed by GitHub
parent b5235fca2e
commit 06ee2d8433
14 changed files with 744 additions and 27 deletions
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from typing import Any
import regex as re
@@ -42,16 +43,57 @@ def humming_is_layer_skipped(config: dict[str, Any], prefix: str):
return False
def convert_linear_layer_to_humming_standard(
layer: LinearBase, name_map: dict[str, str]
):
"""Rename/reshape a linear layer's quantized params (the canonical MPLinear
layout: ``weight_packed`` int32 + ``weight_scale``) into the parameter names
and layout humming's weight schema expects (``weight`` / ``weight_scale``)."""
for name, checkpoint_name in name_map.items():
tensor = getattr(layer, checkpoint_name)
delattr(layer, checkpoint_name)
if name == "weight":
input_dim = getattr(tensor, "input_dim", 1)
output_dim = getattr(tensor, "output_dim", 0)
if input_dim == 0 and output_dim == 1:
tensor = tensor.transpose(1, 0).contiguous()
else:
assert output_dim == 0 and input_dim == 1
tensor = tensor.view(tensor.size(0), -1).view(torch.int32)
elif name in ["weight_scale", "zero_point"]:
if getattr(tensor, "output_dim", 0) == 1:
tensor = tensor.transpose(0, 1).contiguous()
if tensor.ndim == 1:
tensor = tensor.unsqueeze(1)
tensor = tensor.view(torch.int32) if name == "zero_point" else tensor
if isinstance(tensor, torch.nn.Parameter):
param = tensor
else:
param = torch.nn.Parameter(tensor, requires_grad=False)
setattr(layer, name, param)
def prepare_humming_layer(layer: LinearBase, quant_config: dict):
weight_schema = BaseWeightSchema.from_config(quant_config)
input_schema = HummingInputSchema()
shape_k_stacks = [layer.input_size_per_partition]
# ReplicatedLinear has no TP partitioning and so does not set
# input_size_per_partition; for it that is just input_size.
input_size_per_partition = getattr(
layer, "input_size_per_partition", layer.input_size
)
shape_k_stacks = [input_size_per_partition]
shape_n_stacks = layer.output_partition_sizes
# Step 1: convert weight to humming standard format
weight_schema, tensors = weight_schema.convert_humming(
tensors=layer.named_parameters(),
tensors=dict(layer.named_parameters()),
shape_n_stacks=shape_n_stacks,
shape_k_stacks=shape_k_stacks,
param_dtype=layer.params_dtype,
@@ -63,23 +105,37 @@ def prepare_humming_layer(layer: LinearBase, quant_config: dict):
delattr(layer, name)
for name, tensor in tensors.items():
if isinstance(tensor, torch.nn.Parameter):
tensor = tensor.data
param = torch.nn.Parameter(tensor, requires_grad=False)
setattr(layer, name, param)
# Step 2: transform weight (humming standard format) for forwarding
HummingMethod.prepare_layer_meta(
layer=layer,
shape_n=layer.output_partition_sizes_sum,
shape_k=layer.input_size_per_partition,
shape_n=sum(layer.output_partition_sizes),
shape_k=input_size_per_partition,
weight_schema=weight_schema,
input_schema=input_schema,
pad_n_to_multiple=256,
pad_k_to_multiple=128,
has_bias=layer.has_bias,
torch_dtype=layer.param_dtype,
torch_dtype=layer.params_dtype,
)
HummingMethod.transform_humming_layer(layer)
if not hasattr(layer, "locks"):
device = layer.weight.device
locks = torch.zeros(1024, dtype=torch.int32, device=device)
layer.register_buffer("locks", locks)
compute_config = {
"use_batch_invariant": envs.VLLM_BATCH_INVARIANT,
"use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM,
"gemm_type": "dense",
}
layer.compute_config = json.dumps(compute_config)
def prepare_humming_moe_layer(layer: RoutedExperts, quant_config: dict):